Tracts minimal deployment db

.github/workflows/lint.yml

@@ -0,0 +1,39 @@
+name: Lint
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main, staging-* ]
+  workflow_dispatch: 
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: pip cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: lint-pip-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            lint-pip-
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[test]
+
+      - name: Lint
+        run: ./scripts/lint.sh

.github/workflows/test.yml

@@ -0,0 +1,59 @@
+name: Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main, staging-* ]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10']
+        os: [ubuntu-latest]  # , macos-latest]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Ubuntu cache
+        uses: actions/cache@v1
+        if: startsWith(matrix.os, 'ubuntu')
+        with:
+          path: ~/.cache/pip
+          key:
+            ${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ matrix.os }}-${{ matrix.python-version }}-
+
+      - name: macOS cache
+        uses: actions/cache@v1
+        if: startsWith(matrix.os, 'macOS')
+        with:
+          path: ~/Library/Caches/pip
+          key:
+            ${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ matrix.os }}-${{ matrix.python-version }}-
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[dev]
+
+      # - name: Generate databases
+      #   run: python cities/utils/csv_to_db_pipeline.py
+
+      - name: Test
+        run: python -m pytest tests/
+
+      - name: Test Notebooks
+        run: |
+            ./scripts/test_notebooks.sh

.gitignore

@@ -21,3 +21,5 @@ tests/.coverage
 
 *.DS_Store
 .vscode/launch.json
+data/sql/counties_database.db
+data/sql/msa_database.db

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        }
+    ]
+}

Makefile

@@ -1,6 +1,12 @@
 format: FORCE
 	./scripts/clean.sh
 
+
+path ?= .
+
+format_path: FORCE
+	./scripts/clean_path.sh $(path)
+
 lint: FORCE
 	./scripts/lint.sh	
 

cities/__init__.py

@@ -2,4 +2,5 @@
 
 Project short description.
 """
+
 __version__ = "0.0.1"

cities/deployment/tracts_minneapolis/predict.py

@@ -0,0 +1,213 @@
+import copy
+import os
+import time
+
+import dill
+import pandas as pd
+import pyro
+import torch
+from chirho.counterfactual.handlers import MultiWorldCounterfactual
+
+# import chirho
+from chirho.interventional.handlers import do
+from pyro.infer import Predictive
+
+from cities.modeling.zoning_models.zoning_tracts_model import TractsModel
+
+# can be disposed of once you access data in a different manner
+from cities.utils.data_grabber import find_repo_root
+from cities.utils.data_loader import load_sql
+
+root = find_repo_root()
+
+
+#####################
+# data load and prep
+#####################
+
+kwargs = {
+    "categorical": ["year", "census_tract"],
+    "continuous": {
+        "housing_units",
+        "total_value",
+        "median_value",
+        "mean_limit_original",
+        "median_distance",
+        "income",
+        "segregation_original",
+        "white_original",
+    },
+    "outcome": "housing_units",
+}
+
+subset = load_sql(kwargs)
+categorical_levels = {
+    "year": torch.unique(subset["categorical"]["year"]),
+    "census_tract": torch.unique(subset["categorical"]["census_tract"]),
+}
+
+subset_for_preds = copy.deepcopy(subset)
+subset_for_preds["continuous"]["housing_units"] = None
+
+
+########################
+# load trained model (run `train_model.py` first)
+########################
+
+tracts_model = TractsModel(**subset, categorical_levels=categorical_levels)
+
+pyro.clear_param_store()
+
+guide_path = "tracts_model_guide.pkl"
+param_path = "tracts_model_params.pth"
+
+with open(guide_path, "rb") as file:
+    guide = dill.load(file)
+
+pyro.get_param_store().load(param_path)
+
+predictive = Predictive(
+    model=tracts_model,
+    guide=guide,
+    num_samples=100,
+)
+
+
+############################################################
+# define interventions parametrized as in the intended query
+############################################################
+
+
+# these are at the parcel level
+def values_intervention(
+    radius_blue, limit_blue, radius_yellow, limit_yellow, reform_year=2015
+):
+
+    # don't want to load large data multiple times
+    # note we'll need to generate these datasets anew once we switch to the new data pipeline
+
+    if not hasattr(values_intervention, "global_census_ids"):
+        values_intervention.global_census_ids = pd.read_csv(
+            os.path.join(root, "data/minneapolis/processed/census_ids.csv")
+        )
+
+        values_intervention.global_data = pd.read_csv(
+            os.path.join(
+                root,
+                "data/minneapolis/processed/census_tract_intervention_required.csv",
+            )
+        )
+
+        data = values_intervention.global_data
+        census_ids = values_intervention.global_census_ids
+        values_intervention.global_data = data[
+            (data["census_tract"].isin(census_ids["census_tract"]))
+            & (data["year"].isin(census_ids["year"]))
+        ]
+
+    data = values_intervention.global_data.copy()
+
+    intervention = copy.deepcopy(values_intervention.global_data["limit_con"])
+    downtown = data["downtown_yn"]
+    new_blue = (
+        (~downtown)
+        & (data["year"] >= reform_year)
+        & (data["distance_to_transit"] <= radius_blue)
+    )
+    new_yellow = (
+        (~downtown)
+        & (data["year"] >= reform_year)
+        & (data["distance_to_transit"] > radius_blue)
+        & (data["distance_to_transit"] <= radius_yellow)
+    )
+    new_other = (
+        (~downtown)
+        & (data["year"] > reform_year)
+        & (data["distance_to_transit"] > radius_yellow)
+    )
+
+    intervention[downtown] = 0.0
+    intervention[new_blue] = limit_blue
+    intervention[new_yellow] = limit_yellow
+    intervention[new_other] = 1.0
+
+    data["intervention"] = intervention
+
+    return data
+
+
+# generate three interventions at the parcel level
+
+start = time.time()
+simple_intervention = values_intervention(300, 0.5, 700, 0.7, reform_year=2015)
+end = time.time()
+print("Time to run values_intervention 1: ", end - start)
+start2 = time.time()
+simple_intervention2 = values_intervention(400, 0.5, 800, 0.6, reform_year=2013)
+end2 = time.time()
+print("Time to run values_intervention 2: ", end2 - start2)
+start3 = time.time()
+simple_intervention3 = values_intervention(200, 0.4, 1000, 0.65, reform_year=2013)
+end3 = time.time()
+print("Time to run values_intervention 3: ", end3 - start3)
+
+
+# these are at the tracts level
+
+
+def tracts_intervention(
+    radius_blue, limit_blue, radius_yellow, limit_yellow, reform_year=2015
+):
+
+    parcel_interventions = values_intervention(
+        radius_blue, limit_blue, radius_yellow, limit_yellow, reform_year=reform_year
+    )
+
+    aggregate = (
+        parcel_interventions[["census_tract", "year", "intervention"]]
+        .groupby(["census_tract", "year"])
+        .mean()
+        .reset_index()
+    )
+
+    if not hasattr(tracts_intervention, "global_census_ids"):
+
+        tracts_intervention.global_valid_pairs = set(
+            zip(
+                values_intervention.global_census_ids["census_tract"],
+                values_intervention.global_census_ids["year"],
+            )
+        )
+
+    subaggregate = aggregate[
+        aggregate[["census_tract", "year"]]
+        .apply(tuple, axis=1)
+        .isin(tracts_intervention.global_valid_pairs)
+    ].copy()
+
+    return torch.tensor(list(subaggregate["intervention"]))
+
+
+# generate two interventions at the tracts level
+
+start = time.time()
+t_intervention = tracts_intervention(300, 0.5, 700, 0.7, reform_year=2015)
+end = time.time()
+print("Time to run tracts_intervention 1: ", end - start)
+
+start2 = time.time()
+t_intervention2 = tracts_intervention(400, 0.5, 800, 0.6, reform_year=2013)
+end2 = time.time()
+print("Time to run tracts_intervention 2: ", end2 - start2)
+
+
+##################################
+# use interventions with the model
+##################################
+
+with MultiWorldCounterfactual() as mwc:
+    with do(actions={"limit": torch.tensor(0.0)}):
+        samples = predictive(**subset_for_preds)
+
+
+assert samples["limit"].shape[:-1] == torch.Size([100, 2, 1, 1, 1])