Skip to content

Commit

Permalink
merge release/2.13
Browse files Browse the repository at this point in the history
  • Loading branch information
CI committed Apr 13, 2020
2 parents 8c0d0fb + 5887ec1 commit 40458f8
Show file tree
Hide file tree
Showing 53 changed files with 30,123 additions and 18,454 deletions.
10 changes: 10 additions & 0 deletions CHANGES
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
2.13
----
* Add new office endpoint
* Update etools source to match latest etools
* Add ordering to model Meta for pagination where needed

2.12
----
* Add disaggregation field to PRP data report

2.11
----
* fixes #18276 - Add `Achievement in reporting period` field to the prp/datareport endpoint
Expand Down
43 changes: 19 additions & 24 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,35 +1,33 @@
VERSION=2.0.0
BUILDDIR?='~build'
PYTHONPATH:=${PWD}/tests/:${PWD}
BUILDDIR?='build'
PG_ETOOLS_PARAMS?=-U postgres -p 15432 -h 127.0.0.1
PGHOST=127.0.0.1
PGUSER=postgres

.mkbuilddir:
mkdir -p ${BUILDDIR}

help:
echo ""
@echo ""
@echo "Usage:"
@echo " make develop build dev environment "
@echo " make test run tests "
@echo " make lint run lint checks "
@echo " make clean remove generated files "
@echo " make fullclean clean + remove cache "
@echo " make docs generate docs "
@echo " make run run app "


develop:
@poetry run
poetry run pre-commit install
poetry run pre-commit install --hook-type pre-push.
$(MAKE) .init-db


.init-db:
psql -h ${PGHOST} -c 'DROP DATABASE IF EXISTS etools_datamart;' -U ${PGUSER}
psql -h ${PGHOST} -c 'CREATE DATABASE etools_datamart;' -U $PGUSER

test:
pipenv run py.test -v --create-db
poetry run py.test -v --create-db

lint:
pipenv run pre-commit run --all-files
pipenv run pre-commit run --all-files --hook-stage push
pipenv run pre-commit run --all-files --hook-stage manual
poetry run pre-commit run --all-files
poetry run pre-commit run --all-files --hook-stage push
poetry run pre-commit run --all-files --hook-stage manual

clean:
rm -fr ${BUILDDIR} dist *.egg-info .coverage coverage.xml .eggs
Expand All @@ -38,22 +36,19 @@ clean:
rm -f db/clean.sql db/etools.dump db/public.sqldump db/tenant.sql

fullclean:
rm -fr .tox .cache .pytest_cache .venv
rm -fr .cache .pytest_cache
$(MAKE) clean

sync-etools:
sh src/etools_datamart/apps/multitenant/postgresql/dump.sh ${PG_ETOOLS_PARAMS}

docs: .mkbuilddir
docs:
.mkbuilddir
mkdir -p ${BUILDDIR}/docs
sphinx-build -aE docs/ ${BUILDDIR}/docs
ifdef BROWSE
firefox ${BUILDDIR}/docs/index.html
endif

stack:
PYTHONPATH=./src poetry run celery worker -A etools_datamart --loglevel=DEBUG --concurrency=4 --purge --pidfile celery.pid &
PYTHONPATH=./src poetry run celery beat -A etools_datamart.celery --loglevel=DEBUG --pidfile beat.pid &
# PYTHONPATH=./src pipenv run gunicorn -b 0.0.0.0:8000 etools_datamart.config.wsgi --pid gunicorn.pid &
poetry run docker run -d -p 5555:5555 -e CELERY_BROKER_URL=${CELERY_BROKER_URL} --name datamart-flower --rm saxix/flower

run:
docker-compose up
32 changes: 17 additions & 15 deletions db/update_etools_schema.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@ set -e
CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
PROJECT_DIR="`cd "${CURDIR}/..";pwd`"
DUMP_DIRECTORY="$PROJECT_DIR/src/etools_datamart/apps/multitenant/postgresql"
MODEL_DIR="$PROJECT_DIR/src/etools_datamart/apps/etools/models/"
MODEL_DIR="$PROJECT_DIR/src/etools_datamart/apps/sources/etools/models/"

export PGHOST=127.0.0.1
export PGPORT=15432
export DEBUG=True
export PGHOST=db-etools
export PGPORT=5432
export DATABASE_NAME=etools
export DATABASE_USER=postgres
export DATABASE_PASS=
Expand Down Expand Up @@ -235,9 +236,10 @@ function obfuscate(){
return
fi
fi
IFS=,
for tenant in $BASE_SCHEMAS; do
cat clean.tpl.sql | sed "s/_SCHEMA_/${tenant}/" > $CURDIR/clean.sql || exit 1
psql -h ${PGHOST} -p ${PGPORT} \
psql -h ${PGHOST} -p ${PGPORT} -U ${DATABASE_USER} \
-qtAX \
-d ${DATABASE_NAME} \
-f $CURDIR/clean.sql || exit 1
Expand All @@ -252,7 +254,7 @@ function reset_password(){
return
fi
fi
psql -h ${PGHOST} -p ${PGPORT} \
psql -h ${PGHOST} -p ${PGPORT} -U ${DATABASE_USER} \
-qtAX \
-d ${DATABASE_NAME} \
-c "SET search_path=public;UPDATE auth_user SET password='';" || exit 1
Expand All @@ -269,6 +271,7 @@ function dump_public(){
fi
pg_dump --inserts -O \
-d ${DATABASE_NAME} \
-U ${DATABASE_USER} \
-n public \
--format c \
--blobs \
Expand Down Expand Up @@ -315,7 +318,7 @@ function dump_tenant(){
IFS=,
for tenant in $BASE_SCHEMAS; do
echo "4.2.1 Dump $tenant"
pg_dump --inserts -O -d ${DATABASE_NAME} \
pg_dump --inserts -O -d ${DATABASE_NAME} -U ${DATABASE_USER} \
--exclude-table-data django_migrations \
--exclude-table-data django_comments \
--exclude-table-data django_comment_flags \
Expand Down Expand Up @@ -363,10 +366,10 @@ function inspect(){
fi
cd $CURDIR/..
echo "6.1 Inspect 'public' schema"
./manage.py inspectschema --database etools > $CURDIR/../src/etools_datamart/apps/etools/models/public_new.py
./manage.py inspectschema --database etools > $MODEL_DIR/public_new.py

echo "6.2 Inspect 'tenant' schema (${BASE_SCHEMA})"
./manage.py inspectschema --database etools --schema=${BASE_SCHEMA} > $CURDIR/../src/etools_datamart/apps/etools/models/tenant_new.py
./manage.py inspectschema --database etools --schema=${BASE_SCHEMA} > $MODEL_DIR/tenant_new.py

echo "6.3 Backup old models"
mv $MODEL_DIR/public.py $MODEL_DIR/public_old.py
Expand All @@ -375,6 +378,7 @@ function inspect(){
echo "6.4 Enable new models"
mv $MODEL_DIR/public_new.py $MODEL_DIR/public.py
mv $MODEL_DIR/tenant_new.py $MODEL_DIR/tenant.py

echo "6.5 Checking installation"
./manage.py check
cd $CURDIR
Expand All @@ -398,7 +402,7 @@ function summary(){
partners_intervention_flat_locations \
reports_appliedindicator
do
v=`psql -h ${PGHOST} -p ${PGPORT} \
v=`psql -h ${PGHOST} -p ${PGPORT} -U ${DATABASE_USER} \
-qtAX \
-d ${DATABASE_NAME} \
-c "SET search_path=${BASE_SCHEMAS};SELECT COUNT(*) FROM $TABLE;"`
Expand All @@ -408,21 +412,21 @@ function summary(){
done
echo "================================================================"

# v=`psql -h ${PGHOST} -p ${PGPORT} \
# v=`psql -h ${PGHOST} -p ${PGPORT} -U ${DATABASE_USER} \
# -qtAX \
# -d ${DATABASE_NAME} \
# -c "SET search_path=${BASE_SCHEMA};SELECT COUNT(*) FROM partners_partnerorganization;"`
# echo "number_of_partnerorganization = $v"
# echo $v > $PROJECT_DIR/tests/PARTNERORGANIZATION
#
# v=`psql -h ${PGHOST} -p ${PGPORT} \
# v=`psql -h ${PGHOST} -p ${PGPORT} -U ${DATABASE_USER} \
# -qtAX \
# -d ${DATABASE_NAME} \
# -c "SET search_path=${BASE_SCHEMA};SELECT COUNT(*) FROM partners_intervention;"`
# echo $v > $PROJECT_DIR/tests/INTERVENTION
# echo "number_of_intervention = $v"
#
# v=`psql -h ${PGHOST} -p ${PGPORT} \
# v=`psql -h ${PGHOST} -p ${PGPORT} -U ${DATABASE_USER} \
# -qtAX \
# -d ${DATABASE_NAME} \
# -c "SET search_path=${BASE_SCHEMA};SELECT COUNT(*) FROM activities_activity;"`
Expand All @@ -444,8 +448,6 @@ function clean(){
rm -f tenant_*.sql
rm -f clean.sql
rm -f public.sqldump


}

start=$SECONDS
Expand All @@ -454,7 +456,7 @@ start=$SECONDS
if [[ "$DROP" == "1" ]]; then
drop
else
echo "1.x SKIP Dropping ans recreating database (RESTORE)"
echo "1.x SKIP Dropping and recreating database (RESTORE)"
fi

if [[ "$OBFUSCATE" == "1" ]]; then
Expand Down
132 changes: 70 additions & 62 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -1,83 +1,91 @@
version: '2'
services:
# proxy:
# build: ./proxy
# ports:
# - 8000:8000
# depends_on:
# - datamart
# - flower

# base:
# build:
# context: .
# dockerfile: ./docker/Dockerfile.alpine.base
# args:
# VERSION: "x.x"
proxy:
image: traefik:v2.1
command: --api.insecure=True --providers.docker
ports:
- "8083:80"
- "8080:8080"
container_name: datamart_proxy
volumes:
- /var/run/docker.sock:/var/run/docker.sock

# datamart:
# build:
# context: .
# dockerfile: ./docker/Dockerfile.alpine
# args:
# BASE_IMAGE: "2.11-base"
# environment: &environment
# AUTOCREATE_USERS: "admin,123"
# BASE_IMAGE: "compose"
# CACHE_URL: "redis://redis:6379/1"
# CACHE_URL_LOCK: "redis://redis:6379/1"
# CACHE_URL_API: "redis://redis:6379/1"
# CACHE_URL_TEMPLATE: "redis://redis:6379/1"
# CELERY_BROKER_URL: "redis://redis:6379/1"
# CELERY_RESULT_BACKEND: "redis://redis:6379/1"
## CELERY_BROKER_URL: "amqp://guest:guest@rabbitmq/datamart"
## CELERY_RESULT_BACKEND: "amqp://guest:guest@rabbitmq/datamart"
# CSRF_COOKIE_SECURE: 0
# DATABASE_URL: postgis://postgres:@db:5432/etools_datamart
# DATABASE_URL_ETOOLS: postgis://postgres:@etools:5432/etools
# DEBUG: 0
# SECRET_KEY: kuhfjhgfuytfuytfuygfuytdfuydfuygdfuygdfuytf
# SECURE_BROWSER_XSS_FILTER: 0
# SECURE_CONTENT_TYPE_NOSNIFF: 0
# SECURE_FRAME_DENY: 0
# SECURE_HSTS_PRELOAD: 0
# SECURE_HSTS_SECONDS: 0
# SECURE_SSL_REDIRECT: 0
# SESSION_COOKIE_HTTPONLY: 0
# SESSION_COOKIE_SECURE: 0
# STATIC_ROOT: /var/datamart/static/
# STATIC_URL: /dm-static/
# command: datamart
# ports:
# - 9999:8000
# depends_on:
# - db
# - etools
# - redis
datamart:
build:
context: .
dockerfile: ./docker/Dockerfile.dev
args:
BASE_IMAGE: "2.11-base"
container_name: datamart
environment:
DEBUG: 1
DATABASE_URL: postgis://postgres:@db:5432/etools_datamart
DATABASE_URL_ETOOLS: postgis://postgres:@db-etools:5432/etools
DATABASE_URL_PRP: postgis://postgres:@db-prp:5432/prp
AUTOCREATE_USERS: "admin,123"
BASE_IMAGE: "compose"
CACHE_URL: "redis://redis:6379/1"
CACHE_URL_LOCK: "redis://redis:6379/1"
CACHE_URL_API: "redis://redis:6379/1"
CACHE_URL_TEMPLATE: "redis://redis:6379/1"
CELERY_BROKER_URL: "redis://redis:6379/1"
CELERY_RESULT_BACKEND: "redis://redis:6379/1"
CSRF_COOKIE_SECURE: 0
SECURE_BROWSER_XSS_FILTER: 0
SECURE_CONTENT_TYPE_NOSNIFF: 0
SECURE_FRAME_DENY: 0
SECURE_HSTS_PRELOAD: 0
SECURE_SSL_REDIRECT: 0
SESSION_COOKIE_HTTPONLY: 0
SESSION_COOKIE_SECURE: 0
STATIC_ROOT: /code/src/etools_datamart/apps/web/static/
GEOS_LIBRARY_PATH: "/usr/lib/libgeos_c.so.1"
GDAL_LIBRARY_PATH: "/usr/lib/libgdal.so.26"
command: bash -c "python /code/manage.py migrate && python /code/manage.py runserver 0.0.0.0:8080"
volumes:
- "$PWD:/code"
labels:
- "traefik.http.routers.datamart.rule=PathPrefix(`/`)"
- traefik.http.routers.datamart.service=datamart
- traefik.http.services.datamart.loadBalancer.server.port=8080
- traefik.enable=true
depends_on:
- db
- redis

db:
image: mdillon/postgis:9.6
container_name: datamart_db
environment:
POSTGRES_PASSWORD:
POSTGRES_USER: postgres
POSTGRES_DB: etools_datamart
volumes:
- "$PWD/~build/db:/var/lib/postgresql/data"
ports:
- 25432:5432
- "$PWD/build/db:/var/lib/postgresql/data"

etools:
# Rely on etools and prp database instances running locally
db-etools:
image: mdillon/postgis:9.6
ports:
- "15432:5432"
container_name: datamart_etools
shm_size: '1gb'
environment:
POSTGRES_PASSWORD:
POSTGRES_USER: postgres
POSTGRES_PASSWORD:
POSTGRES_DB: etools
volumes:
- ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data
- "$PWD/build/etools:/var/lib/postgresql/data"

db-prp:
image: mdillon/postgis:9.6
shm_size: '1gb'
container_name: datamart_prp
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD:
POSTGRES_DB: prp
volumes:
- "$PWD/build/prp:/var/lib/postgresql/data"

redis:
image: redis:alpine

container_name: datamart_redis
1 change: 1 addition & 0 deletions docker/Dockerfile.alpine.base
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ RUN apk add --no-cache --virtual .run-deps \
# libzmq \
# jpeg \
mailcap \
postgresql-client \
# xmlsec \
zlib

Expand Down
15 changes: 15 additions & 0 deletions docker/Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
ARG BASE_IMAGE
FROM unicef/datamart:${BASE_IMAGE}

Add poetry.lock .
Add pyproject.toml .

RUN apk add bash
RUN mkdir /tmp/static

ENV PYTHONUNBUFFERED 1
ENV PYTHONPATH /code
ENV DJANGO_SETTINGS_MODULE etools_datamart.config.settings

VOLUME "./:/code/"
WORKDIR /code/
Loading

0 comments on commit 40458f8

Please sign in to comment.