Skip to content

Commit 29ecbd0

Browse files
andreas-paulandreas-paul
committed
Initial commit
0 parents  commit 29ecbd0

21 files changed

+909
-0
lines changed

‎.dockerignore

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Custom
2+
data/
3+
cache/
4+
.vscode
5+
__pycache__
6+
src/__pycache__
7+
tests/__pycache__

‎.gitignore

+157
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# Custom
2+
data/
3+
cache/
4+
.vscode
5+
6+
# Byte-compiled / optimized / DLL files
7+
__pycache__/
8+
*.py[cod]
9+
*$py.class
10+
11+
# C extensions
12+
*.so
13+
14+
# Distribution / packaging
15+
.Python
16+
build/
17+
develop-eggs/
18+
dist/
19+
downloads/
20+
eggs/
21+
.eggs/
22+
lib/
23+
lib64/
24+
parts/
25+
sdist/
26+
var/
27+
wheels/
28+
share/python-wheels/
29+
*.egg-info/
30+
.installed.cfg
31+
*.egg
32+
MANIFEST
33+
34+
# PyInstaller
35+
# Usually these files are written by a python script from a template
36+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
37+
*.manifest
38+
*.spec
39+
40+
# Installer logs
41+
pip-log.txt
42+
pip-delete-this-directory.txt
43+
44+
# Unit test / coverage reports
45+
htmlcov/
46+
.tox/
47+
.nox/
48+
.coverage
49+
.coverage.*
50+
.cache
51+
nosetests.xml
52+
coverage.xml
53+
*.cover
54+
*.py,cover
55+
.hypothesis/
56+
.pytest_cache/
57+
cover/
58+
59+
# Translations
60+
*.mo
61+
*.pot
62+
63+
# Django stuff:
64+
*.log
65+
local_settings.py
66+
db.sqlite3
67+
db.sqlite3-journal
68+
69+
# Flask stuff:
70+
instance/
71+
.webassets-cache
72+
73+
# Scrapy stuff:
74+
.scrapy
75+
76+
# Sphinx documentation
77+
docs/_build/
78+
79+
# PyBuilder
80+
.pybuilder/
81+
target/
82+
83+
# Jupyter Notebook
84+
.ipynb_checkpoints
85+
86+
# IPython
87+
profile_default/
88+
ipython_config.py
89+
90+
# pyenv
91+
# For a library or package, you might want to ignore these files since the code is
92+
# intended to run in multiple environments; otherwise, check them in:
93+
# .python-version
94+
95+
# pipenv
96+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
97+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
98+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
99+
# install all needed dependencies.
100+
#Pipfile.lock
101+
102+
# poetry
103+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
104+
# This is especially recommended for binary packages to ensure reproducibility, and is more
105+
# commonly ignored for libraries.
106+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
107+
#poetry.lock
108+
109+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
110+
__pypackages__/
111+
112+
# Celery stuff
113+
celerybeat-schedule
114+
celerybeat.pid
115+
116+
# SageMath parsed files
117+
*.sage.py
118+
119+
# Environments
120+
.env
121+
.venv
122+
env/
123+
venv/
124+
ENV/
125+
env.bak/
126+
venv.bak/
127+
128+
# Spyder project settings
129+
.spyderproject
130+
.spyproject
131+
132+
# Rope project settings
133+
.ropeproject
134+
135+
# mkdocs documentation
136+
/site
137+
138+
# mypy
139+
.mypy_cache/
140+
.dmypy.json
141+
dmypy.json
142+
143+
# Pyre type checker
144+
.pyre/
145+
146+
# pytype static type analyzer
147+
.pytype/
148+
149+
# Cython debug symbols
150+
cython_debug/
151+
152+
# PyCharm
153+
# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can
154+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
155+
# and can be added to the global gitignore or merged into this file. For a more nuclear
156+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
157+
#.idea/

‎Dockerfile

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM continuumio/miniconda3
2+
3+
WORKDIR /
4+
5+
COPY environment.yml .
6+
RUN conda env create -f environment.yml
7+
8+
RUN echo "source activate venv-axelspace-assignment" > ~/.bashrc
9+
ENV PATH /opt/conda/envs/venv-axelspace-assignment/bin:$PATH
10+
11+
RUN echo "Testing imports"
12+
RUN ["python", "-c", "import geopandas"]
13+
RUN echo "Import success"
14+
15+
COPY . .
16+
17+
RUN echo "Starting test suite"
18+
RUN ["python", "-m", "pytest", "tests/"]
19+
RUN echo "Test suite completed successfully"
20+
21+
ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "venv-axelspace-assignment", "PYTHONPATH=${PWD}:${PWD}/src", "python", "./solution.py"]

‎README.md

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
## INTRODUCTION
2+
3+
This repository contains code to:
4+
5+
- Download Landsat imagery from Amazon Web Services open data repository for the San Francisco area
6+
- Download OpenStreetMap road data for San Francisco (tagged 'San Francisco')
7+
- Create RGB composite raster from Landsat data, with additional pan-sharpening (_**tci.jpg**_)
8+
- Create bounding box and clip RGB composite raster
9+
- Create a map zoomed in to San Francisco city with overlain road network (_**map.jpg**_)
10+
- Dockerize all related code items described above
11+
- Perform unit testing of all related code items described above
12+
13+
It was developed in Linux through WSL2 (Ubuntu 20.04) on a Windows10 machine, and tested on Linux and Windows 10 (powershell), so all of it should run fine on either of these systems. It is likely to run fine on MacOS as well, but this has not been tested.
14+
15+
Total processing time on a standard PC is approximately 3 minutes. Depending on the system, this can be less or more. If it is running longer than 10 minutes, please check internet speeds and/or try on a different machine.
16+
17+
The code creates two directories in its working directory: _**tmp**_ and _**out**_:
18+
19+
- The _**tmp**_ directory contains all data that was used to create the required products; this includes satellite imagery and vector data.
20+
21+
- The _**out**_ directory contains the two data products, **tci.jpg** and **map.jpg**.
22+
23+
Please note, that the road network shown in **map.jpg** only covers San Francisco proper until the county borders. Because of that, it does not extent fully in the lower (southern) part of the image.
24+
25+
## HOW TO RUN
26+
27+
### Docker
28+
29+
This build process has been tested on Linux via WSL2 and Windows 10 (cmd; powershell).
30+
31+
To build the Docker image, run the following command from within the repository:
32+
33+
```bash
34+
docker build -t paul_axelspace_assignment .
35+
```
36+
37+
The test suite will run during the build process, using _**pytest**_. If it fails, the Docker image build will fail as well. Please open an issue if this happens.
38+
39+
Once the build process is complete, run the Docker image with the following command:
40+
41+
```bash
42+
docker run --rm -i -v ${PWD}:/app -e AWS_ACCESS_KEY_ID=<access-key-id> -e AWS_SECRET_ACCESS_KEY=<secret-access-key> paul_axelspace_assignment
43+
```
44+
45+
Please do not forget to replace "access-key-id" and "secret-access-key" with valid AWS credentials. Without these, the program will download a backup file from my dropbox.
46+
47+
### Testing
48+
49+
The test suite is run during the build process of the docker image. However, it can also be directly invoked by running _**pytest**_ from within the main directory of the repo.
50+
51+
If run outside of docker, the test suite requires an existing conda environment. This environment can be created using the conda _environment.yml_ file present in the repository. The easiest way to get a working conda installation is by installing [Miniconda](https://docs.conda.io/en/latest/miniconda.html).
52+
53+
Once conda is installed, run the following command to create a new conda environment:
54+
55+
```bash
56+
conda env create -f environment.yml
57+
```
58+
59+
Activate the conda environment by running:
60+
61+
```bash
62+
conda activate paul_axelspace_assignment
63+
```
64+
65+
Once the environment is activated, run the following command from the main folder of the repository, to start the testing suite:
66+
67+
```bash
68+
pytest
69+
```
70+
71+
### Bash
72+
73+
If the conda environment has been created, the code can also be run directly:
74+
75+
```bash
76+
python solution.py
77+
```
78+
79+
## DATA SOURCES
80+
81+
- Landsat 8 imagery courtesy of the [U.S. Geological Survey](https://www.usgs.gov)
82+
83+
- OpenStreetMap data courtesy of OpenStreetMap contributors, and available via [OpenStreetMap.org](https://www.openstreetmap.org), license under the [Open Data Commons](https://opendatacommons.org/licenses/odbl/) license.

‎environment.yml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
name: venv-axelspace-assignment
2+
channels:
3+
- conda-forge
4+
- defaults
5+
dependencies:
6+
- python>=3.10
7+
- boto3>=1.18.21
8+
- fiona>=1.8.20
9+
- geopandas>=0.10.2
10+
- loguru>=0.5.3
11+
- osmnx>=1.1.2
12+
- pandas>=1.3.5
13+
- pydantic>=1.9.0
14+
- pytest>=6.2.5
15+
- rasterio>=1.2.10

‎pytest.ini

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[pytest]
2+
addopts = -p no:warnings

‎solution.py

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import os
2+
from src.bbox import create_bounding_box
3+
from src.config import log, settings
4+
from src.download import get_landsat_data
5+
from src.mapping import overlay_road_network
6+
from src.osm import get_osm_street_data
7+
from src.processing import rgb_pansharpening, save_raster_as_jpg, clip_raster, color_correct
8+
9+
10+
def main():
11+
12+
log.info('Starting solution. This will take a few minutes.')
13+
14+
# Define initial variables
15+
log.info('Defining initial variables')
16+
work_dir = settings.workdir
17+
out_dir = settings.outdir
18+
rgb_img_id = f"{work_dir}/{settings.image_id}_PAN.TIF"
19+
bbox_path = f'{work_dir}/San-Francisco-bbox.gpkg'
20+
road_path = f'{work_dir}/San-Francisco-roads.gpkg'
21+
22+
# Create directories.
23+
log.info('Creating directories')
24+
for directory in [work_dir, out_dir]:
25+
if not os.path.exists(directory):
26+
os.mkdir(directory)
27+
28+
# Download Landsat data
29+
log.info('Downloading Landsat data')
30+
get_landsat_data(image_id=settings.image_id,
31+
bucket = settings.s3_bucket,
32+
path = settings.s3_path,
33+
bands = settings.use_bands)
34+
35+
# Download OpenStreetMap data
36+
log.info('Downloading OpenStreetMap road data')
37+
get_osm_street_data(place="San Francisco, CA, USA",
38+
save_file_path=road_path)
39+
40+
# Stack and pansharpen Landsat imagery
41+
log.info('Stacking and pansharpening RGB raster')
42+
rgb_pansharpening(save_file_path=rgb_img_id,
43+
image_id=settings.image_id)
44+
45+
# Save pansharpened RGB image as 'tci.jpg'
46+
log.info('Saving pansharpened RGB image to disk')
47+
save_raster_as_jpg(src_file_path=rgb_img_id,
48+
save_file_path=f'{work_dir}/_tmp_tci.jpg')
49+
log.success('Success')
50+
51+
# Color correct RGB image
52+
log.info('Applying color corrections')
53+
color_correct(src_file_path=f'{work_dir}/_tmp_tci.jpg',
54+
out_file_path=f'{out_dir}/tci.jpg')
55+
56+
# Create bounding box to clip raster
57+
log.info('Creating bounding box from coordinates')
58+
create_bounding_box(save_file_path=bbox_path,
59+
latitudes=settings.latitudes,
60+
longitudes=settings.longitudes,
61+
crs=settings.crs)
62+
63+
# Clip raster to San Francisco area
64+
log.info('Clipping raster to San Francisco area')
65+
clip_raster(extent_file_path=bbox_path,
66+
src_raster_path=f'{out_dir}/tci.jpg',
67+
out_raster_path=f'{work_dir}/_tmp_map.tif')
68+
69+
# Overlay road network
70+
log.info('Overlaying road network and saving map to disk')
71+
overlay_road_network(road_network=road_path,
72+
src_file_path=f'{work_dir}/_tmp_map.tif',
73+
out_file_path=f'{out_dir}/map.jpg')
74+
75+
# Final check
76+
assert os.path.exists(f'{out_dir}/tci.jpg')
77+
assert os.path.exists(f'{out_dir}/map.jpg')
78+
79+
log.success('Finished solution. Thank you for your patience.')
80+
81+
82+
if __name__ == "__main__":
83+
main()

‎src/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)