Skip to content

Commit

Permalink
Merge pull request #862 from CodeForPhilly/staging
Browse files Browse the repository at this point in the history
Merge staging into main
  • Loading branch information
nlebovits authored Aug 7, 2024
2 parents 59f32e0 + 22fe886 commit 070084c
Show file tree
Hide file tree
Showing 21 changed files with 3,481 additions and 3,255 deletions.
27 changes: 12 additions & 15 deletions .github/workflows/pr_checks_backend.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ on:
- 'Dockerfile-pg'
- 'init_pg.sql'
- 'docker-compose.yml'
workflow_dispatch:

jobs:
build:
Expand All @@ -29,46 +30,42 @@ jobs:
with:
python-version: '3.11.4'

- name: Run Black Formatter in Docker
- name: Run Ruff Formatter in Docker
run: |
cd data
docker-compose run --rm formatter
continue-on-error: true

- name: Run Flake8
- name: Run Ruff Linter in Docker
run: |
cd data
pip install flake8
flake8 .
docker-compose run --rm linter
continue-on-error: true

- name: Install dependencies and build Docker images
- name: Build vacant-lots-proj
run: |
cd data
docker-compose pull
docker-compose build
docker-compose build vacant-lots-proj
- name: Run vacant-lots-proj
- name: Build data-fetcher
run: |
cd data
docker-compose up -d vacant-lots-proj
continue-on-error: true
docker-compose build data-fetcher
- name: Build and check Postgres container
- name: Run all services
run: |
cd data
docker-compose build postgres
continue-on-error: true
docker-compose up -d
- name: Check build status
if: failure()
run: |
echo "The vacant-lots-proj or postgres container failed to build and run."
echo "One or more services failed to build and run."
exit 1
- name: Report success
if: success()
run: echo "The vacant-lots-proj and postgres container built and ran successfully."
run: echo "All services built and ran successfully."

- name: Check formatter and linter status
if: failure()
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ data.env
.idea
data/src/tmp
.DS_Store
/data/src/local_outputs/

## App

Expand Down
51 changes: 51 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
repos:
# Python hooks
- repo: local
hooks:
# Ruff for Python linting
- id: ruff
name: ruff (Python linting)
entry: ruff check --fix
language: python
types_or: [python, pyi, jupyter]
files: \.py$
exclude: ^data/src/awkde/

# Ruff for Python formatting
- id: ruff-format
name: ruff-format (Python formatting)
entry: ruff format
language: python
types_or: [python, pyi, jupyter]
files: \.py$
exclude: ^data/src/awkde/

# Dependency checks for Python
- id: check-python-dependencies
name: Check Python dependencies
entry: bash -c 'cd data/src && pipenv check --bare --quiet || (echo "Python dependencies check failed" && exit 1)'
language: system
files: ^data/src/Pipfile$

# JavaScript hooks
- repo: local
hooks:
# Formatting with Prettier
- id: prettier
name: Prettier (Code formatting)
entry: npm run format
language: node
files: \.(js|jsx|ts|tsx)$

# Linting with ESLint
- id: eslint
name: ESLint (JavaScript linting)
entry: npm run lint
language: node
files: \.(js|jsx|ts|tsx)$

# Dependency checks for Node.js
- id: npm-audit
name: Check Node.js dependencies
entry: npm audit
language: node
14 changes: 7 additions & 7 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@
"${workspaceFolder}/data/src",
"${workspaceFolder}/data/src/awkde"
],
"python.testing.pytestEnabled": true,
"python.testing.cwd": "${workspaceFolder}/data/src",
"[python]": {
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.fixAll": "explicit",
"source.organizeImports": "explicit"
},
"editor.defaultFormatter": "ms-python.black-formatter"
"editor.defaultFormatter": "charliermarsh.ruff"
},
"python.formatting.provider": "black",
"python.linting.flake8Enabled": true,
"python.linting.enabled": true,
"python.linting.flake8Args": ["--max-line-length=88"]
"notebook.formatOnSave.enabled": true,
"notebook.codeActionsOnSave": {
"notebook.source.fixAll": "explicit",
"notebook.source.organizeImports": "explicit"
}
}
14 changes: 14 additions & 0 deletions data/Dockerfile-fetch-data
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Use the existing main container image
FROM vacant-lots-proj:latest

# Set the working directory in the container
WORKDIR /usr/src/app

# Copy the fetch_and_save_data.py script to the container
COPY src/fetch_and_save_data.py ./

# Ensure Pipenv and dependencies are installed
RUN pipenv install --ignore-pipfile --dev

# Set the default command to run the script with pipenv and python
ENTRYPOINT ["sh", "-c", "pipenv run python ./fetch_and_save_data.py \"$@\"", "--"]
32 changes: 25 additions & 7 deletions data/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ services:
vacant-lots-proj:
build:
context: .
network: host
dockerfile: Dockerfile
image: vacant-lots-proj:latest
environment:
- GOOGLE_APPLICATION_CREDENTIALS=/app/service-account-key.json
- CFP_MAPBOX_TOKEN_UPLOADER
Expand All @@ -16,30 +17,34 @@ services:
- ~/.config/gcloud/application_default_credentials.json:/app/service-account-key.json
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro

extra_hosts:
- host.docker.internal:host-gateway
network_mode: 'host'

formatter:
build:
context: .
network: host
volumes:
- ./src:/usr/src/app
command: sh -c "pipenv run black /usr/src/app"
command: sh -c "pip install ruff && ruff /usr/src/app --fix --exclude '/usr/src/app/data/src/awkde/'"
network_mode: 'host'

linter:
build:
context: .
volumes:
- ./src:/usr/src/app
command: sh -c "pip install ruff && ruff check /usr/src/app --exclude '/usr/src/app/data/src/awkde/'"
network_mode: 'host'

streetview:
build:
context: .
network: host
environment:
- GOOGLE_APPLICATION_CREDENTIALS=/app/service-account-key.json
- CLEAN_GREEN_GOOGLE_KEY
- VACANT_LOTS_DB
- GOOGLE_CLOUD_BUCKET_NAME

volumes:
- ./src:/usr/src/app
- ~/.config/gcloud/application_default_credentials.json:/app/service-account-key.json
Expand All @@ -55,7 +60,6 @@ services:
build:
context: .
dockerfile: Dockerfile-pg
network: host
environment:
- PGPORT=5433
- POSTGRES_PASSWORD
Expand All @@ -70,5 +74,19 @@ services:
extra_hosts:
- host.docker.internal:host-gateway

data-fetcher:
build:
context: .
dockerfile: Dockerfile-fetch-data
depends_on:
- vacant-lots-proj
environment:
- VACANT_LOTS_DB
volumes:
- ./src:/usr/src/app
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
network_mode: 'host'

volumes:
database_volume:
1 change: 1 addition & 0 deletions data/src/Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ pytest = "*"
networkx = "*"
libpysal = "*"
jenkspy = "*"
pyarrow = "*"

[dev-packages]

Expand Down
Loading

0 comments on commit 070084c

Please sign in to comment.