Merge staging into main

.github/workflows/pr_checks_backend.yml

@@ -9,6 +9,7 @@ on:
       - 'Dockerfile-pg'
       - 'init_pg.sql'
       - 'docker-compose.yml'
+  workflow_dispatch:
 
 jobs:
   build:
@@ -29,46 +30,42 @@ jobs:
         with:
           python-version: '3.11.4'
 
-      - name: Run Black Formatter in Docker
+      - name: Run Ruff Formatter in Docker
         run: |
           cd data
           docker-compose run --rm formatter
         continue-on-error: true
 
-      - name: Run Flake8
+      - name: Run Ruff Linter in Docker
         run: |
           cd data
-          pip install flake8
-          flake8 .
+          docker-compose run --rm linter
         continue-on-error: true
 
-      - name: Install dependencies and build Docker images
+      - name: Build vacant-lots-proj
         run: |
           cd data
-          docker-compose pull
-          docker-compose build
+          docker-compose build vacant-lots-proj
 
-      - name: Run vacant-lots-proj
+      - name: Build data-fetcher
         run: |
           cd data
-          docker-compose up -d vacant-lots-proj
-        continue-on-error: true
+          docker-compose build data-fetcher
 
-      - name: Build and check Postgres container
+      - name: Run all services
         run: |
           cd data
-          docker-compose build postgres
-        continue-on-error: true
+          docker-compose up -d
 
       - name: Check build status
         if: failure()
         run: |
-          echo "The vacant-lots-proj or postgres container failed to build and run."
+          echo "One or more services failed to build and run."
           exit 1
 
       - name: Report success
         if: success()
-        run: echo "The vacant-lots-proj and postgres container built and ran successfully."
+        run: echo "All services built and ran successfully."
 
       - name: Check formatter and linter status
         if: failure()

.gitignore

@@ -7,6 +7,7 @@ data.env
 .idea
 data/src/tmp
 .DS_Store
+/data/src/local_outputs/
 
 ## App
 

.pre-commit-config.yaml

@@ -0,0 +1,51 @@
+repos:
+  # Python hooks
+  - repo: local
+    hooks:
+      # Ruff for Python linting
+      - id: ruff
+        name: ruff (Python linting)
+        entry: ruff check --fix
+        language: python
+        types_or: [python, pyi, jupyter]
+        files: \.py$
+        exclude: ^data/src/awkde/
+
+      # Ruff for Python formatting
+      - id: ruff-format
+        name: ruff-format (Python formatting)
+        entry: ruff format
+        language: python
+        types_or: [python, pyi, jupyter]
+        files: \.py$
+        exclude: ^data/src/awkde/
+
+      # Dependency checks for Python
+      - id: check-python-dependencies
+        name: Check Python dependencies
+        entry: bash -c 'cd data/src && pipenv check --bare --quiet || (echo "Python dependencies check failed" && exit 1)'
+        language: system
+        files: ^data/src/Pipfile$
+
+  # JavaScript hooks
+  - repo: local
+    hooks:
+      # Formatting with Prettier
+      - id: prettier
+        name: Prettier (Code formatting)
+        entry: npm run format
+        language: node
+        files: \.(js|jsx|ts|tsx)$
+
+      # Linting with ESLint
+      - id: eslint
+        name: ESLint (JavaScript linting)
+        entry: npm run lint
+        language: node
+        files: \.(js|jsx|ts|tsx)$
+
+      # Dependency checks for Node.js
+      - id: npm-audit
+        name: Check Node.js dependencies
+        entry: npm audit
+        language: node

.vscode/settings.json

@@ -5,17 +5,17 @@
     "${workspaceFolder}/data/src",
     "${workspaceFolder}/data/src/awkde"
   ],
-  "python.testing.pytestEnabled": true,
-  "python.testing.cwd": "${workspaceFolder}/data/src",
   "[python]": {
     "editor.formatOnSave": true,
     "editor.codeActionsOnSave": {
+      "source.fixAll": "explicit",
       "source.organizeImports": "explicit"
     },
-    "editor.defaultFormatter": "ms-python.black-formatter"
+    "editor.defaultFormatter": "charliermarsh.ruff"
   },
-  "python.formatting.provider": "black",
-  "python.linting.flake8Enabled": true,
-  "python.linting.enabled": true,
-  "python.linting.flake8Args": ["--max-line-length=88"]
+  "notebook.formatOnSave.enabled": true,
+  "notebook.codeActionsOnSave": {
+    "notebook.source.fixAll": "explicit",
+    "notebook.source.organizeImports": "explicit"
+  }
 }

data/Dockerfile-fetch-data

@@ -0,0 +1,14 @@
+# Use the existing main container image
+FROM vacant-lots-proj:latest
+
+# Set the working directory in the container
+WORKDIR /usr/src/app
+
+# Copy the fetch_and_save_data.py script to the container
+COPY src/fetch_and_save_data.py ./
+
+# Ensure Pipenv and dependencies are installed
+RUN pipenv install --ignore-pipfile --dev
+
+# Set the default command to run the script with pipenv and python
+ENTRYPOINT ["sh", "-c", "pipenv run python ./fetch_and_save_data.py \"$@\"", "--"]

data/docker-compose.yml

@@ -2,7 +2,8 @@ services:
   vacant-lots-proj:
     build:
       context: .
-      network: host
+      dockerfile: Dockerfile
+    image: vacant-lots-proj:latest
     environment:
       - GOOGLE_APPLICATION_CREDENTIALS=/app/service-account-key.json
       - CFP_MAPBOX_TOKEN_UPLOADER
@@ -16,30 +17,34 @@ services:
       - ~/.config/gcloud/application_default_credentials.json:/app/service-account-key.json
       - /etc/timezone:/etc/timezone:ro
       - /etc/localtime:/etc/localtime:ro
-
     extra_hosts:
       - host.docker.internal:host-gateway
     network_mode: 'host'
 
   formatter:
     build:
       context: .
-      network: host
     volumes:
       - ./src:/usr/src/app
-    command: sh -c "pipenv run black /usr/src/app"
+    command: sh -c "pip install ruff && ruff /usr/src/app --fix --exclude '/usr/src/app/data/src/awkde/'"
+    network_mode: 'host'
+
+  linter:
+    build:
+      context: .
+    volumes:
+      - ./src:/usr/src/app
+    command: sh -c "pip install ruff && ruff check /usr/src/app --exclude '/usr/src/app/data/src/awkde/'"
     network_mode: 'host'
 
   streetview:
     build:
       context: .
-      network: host
     environment:
       - GOOGLE_APPLICATION_CREDENTIALS=/app/service-account-key.json
       - CLEAN_GREEN_GOOGLE_KEY
       - VACANT_LOTS_DB
       - GOOGLE_CLOUD_BUCKET_NAME
-
     volumes:
       - ./src:/usr/src/app
       - ~/.config/gcloud/application_default_credentials.json:/app/service-account-key.json
@@ -55,7 +60,6 @@ services:
     build:
       context: .
       dockerfile: Dockerfile-pg
-      network: host
     environment:
       - PGPORT=5433
       - POSTGRES_PASSWORD
@@ -70,5 +74,19 @@ services:
     extra_hosts:
       - host.docker.internal:host-gateway
 
+  data-fetcher:
+    build:
+      context: .
+      dockerfile: Dockerfile-fetch-data
+    depends_on:
+      - vacant-lots-proj
+    environment:
+      - VACANT_LOTS_DB
+    volumes:
+      - ./src:/usr/src/app
+      - /etc/timezone:/etc/timezone:ro
+      - /etc/localtime:/etc/localtime:ro
+    network_mode: 'host'
+
 volumes:
   database_volume:

data/src/Pipfile

@@ -29,6 +29,7 @@ pytest = "*"
 networkx = "*"
 libpysal = "*"
 jenkspy = "*"
+pyarrow = "*"
 
 [dev-packages]