From 3fb10cbf0d99f11025bad561f7e515413d19f7c1 Mon Sep 17 00:00:00 2001
From: Nikhil Woodruff <nikhil.woodruff@outlook.com>
Date: Mon, 16 Sep 2024 12:57:56 +0100
Subject: [PATCH] Fix cloud access bugs

---
 .github/workflows/ci_cd.yaml                  | 10 ++++----
 docs/Home.py                                  |  2 ++
 docs/Pages/Methodology.py                     |  2 ++
 docs/Pages/Validation.py                      |  2 ++
 policyengine_uk_data/utils/__init__.py        |  1 +
 .../utils/download_docs_prerequisites.py      | 25 +++++++++++++++++++
 policyengine_uk_data/utils/github.py          |  2 +-
 pyproject.toml                                |  2 +-
 8 files changed, 39 insertions(+), 7 deletions(-)
 create mode 100644 policyengine_uk_data/utils/download_docs_prerequisites.py

diff --git a/.github/workflows/ci_cd.yaml b/.github/workflows/ci_cd.yaml
index b465229..3db31f2 100644
--- a/.github/workflows/ci_cd.yaml
+++ b/.github/workflows/ci_cd.yaml
@@ -63,7 +63,7 @@ jobs:
       - name: Download data inputs
         run: make download
         env:
-          POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
+          POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
       - name: Build datasets
         run: make data
       - name: Run tests
@@ -83,9 +83,9 @@ jobs:
           username: ${{github.actor}}
           password: ${{secrets.GITHUB_TOKEN}}
       - name: Build container
-        run: docker build . -f docker/policyengine_us_data.Dockerfile -t ghcr.io/policyengine/policyengine-us-data:latest
+        run: docker build . -f docker/policyengine_uk_data.Dockerfile -t ghcr.io/policyengine/policyengine-uk-data:latest
       - name: Push container
-        run: docker push ghcr.io/policyengine/policyengine-us-data:latest
+        run: docker push ghcr.io/policyengine/policyengine-uk-data:latest
 
   upload:
     name: Upload Data 
@@ -106,10 +106,10 @@ jobs:
       - name: Download data inputs
         run: make download
         env:
-          POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
+          POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
       - name: Build datasets
         run: make data
       - name: Upload data
         run: make upload
         env:
-          POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
\ No newline at end of file
+          POLICYENGINE_UK_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_UK_DATA_GITHUB_TOKEN }}
\ No newline at end of file
diff --git a/docs/Home.py b/docs/Home.py
index eaa4831..667513d 100644
--- a/docs/Home.py
+++ b/docs/Home.py
@@ -1,4 +1,6 @@
 import streamlit as st
+from policyengine_uk_data.utils.download_docs_prerequisites import download_data
+download_data()
 
 st.set_page_config(layout="wide")
 
diff --git a/docs/Pages/Methodology.py b/docs/Pages/Methodology.py
index b614dd4..e1d178f 100644
--- a/docs/Pages/Methodology.py
+++ b/docs/Pages/Methodology.py
@@ -1,4 +1,6 @@
 import streamlit as st
+from policyengine_uk_data.utils.download_docs_prerequisites import download_data
+download_data()
 
 st.set_page_config(layout="wide")
 
diff --git a/docs/Pages/Validation.py b/docs/Pages/Validation.py
index 7b67485..820a731 100644
--- a/docs/Pages/Validation.py
+++ b/docs/Pages/Validation.py
@@ -1,4 +1,6 @@
 import streamlit as st
+from policyengine_uk_data.utils.download_docs_prerequisites import download_data
+download_data()
 
 st.set_page_config(layout="wide")
 
diff --git a/policyengine_uk_data/utils/__init__.py b/policyengine_uk_data/utils/__init__.py
index 983bc4a..fcdcdb0 100644
--- a/policyengine_uk_data/utils/__init__.py
+++ b/policyengine_uk_data/utils/__init__.py
@@ -2,3 +2,4 @@
 from .uprating import *
 from .datasets import *
 from .loss import *
+from .download_docs_prerequisites import *
\ No newline at end of file
diff --git a/policyengine_uk_data/utils/download_docs_prerequisites.py b/policyengine_uk_data/utils/download_docs_prerequisites.py
new file mode 100644
index 0000000..89fb93b
--- /dev/null
+++ b/policyengine_uk_data/utils/download_docs_prerequisites.py
@@ -0,0 +1,25 @@
+from policyengine_us_data.utils.github import download
+from policyengine_us_data.data_storage import STORAGE_FOLDER
+
+PREREQUISITES = [
+    {
+        "repo": "ukda",
+        "file_name": "frs_2022_23.h5",
+    },
+    {
+        "repo": "ukda",
+        "file_name": "enhanced_frs_2022_23.h5",
+    },
+]
+
+
+def download_data():
+    for prerequisite in PREREQUISITES:
+        if not (STORAGE_FOLDER / prerequisite["file_name"]).exists():
+            download(
+                "PolicyEngine",
+                prerequisite["repo"],
+                "release",
+                prerequisite["file_name"],
+                STORAGE_FOLDER / prerequisite["file_name"],
+            )
diff --git a/policyengine_uk_data/utils/github.py b/policyengine_uk_data/utils/github.py
index c4f2401..43a05e2 100644
--- a/policyengine_uk_data/utils/github.py
+++ b/policyengine_uk_data/utils/github.py
@@ -6,7 +6,7 @@
 import time
 
 auth_headers = {
-    "Authorization": f"token {os.environ.get('POLICYENGINE_US_DATA_GITHUB_TOKEN')}",
+    "Authorization": f"token {os.environ.get('POLICYENGINE_UK_DATA_GITHUB_TOKEN')}",
 }
 
 
diff --git a/pyproject.toml b/pyproject.toml
index 2c4feed..c5e03b5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,7 +27,7 @@ dependencies = [
 dev = [
     "black",
     "pytest",
-    "policyengine_uk @ git+https://github.com/PolicyEngine/policyengine-uk.git@data-update",
+    "policyengine_uk>=1.8.0",
     "streamlit",
 ]