diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 744a13b31..b3e45fc01 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -27,3 +27,4 @@ values = [bumpversion:file:pyproject.toml] +[bumpversion:file:docker-compose.yml] diff --git a/.dockerignore b/.dockerignore index 22429a5a7..9b092d8be 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,4 @@ db docs tests +build diff --git a/.gitignore b/.gitignore index b797096ed..cd2356f0b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .coverage .DS_Store .env +.envrc .idea .pydevproject .tox @@ -41,3 +42,4 @@ xml ~* **/dual-listbox-master/ #src/etools_datamart/apps/security/static/security/dual-listbox-master +build/ diff --git a/db/Makefile b/db/Makefile index a7f08bf05..dbe704321 100644 --- a/db/Makefile +++ b/db/Makefile @@ -1,39 +1,47 @@ +DB_DUMP_LOCATION?=/tmp/psql_data/db2.bz2 +DOCKER_IMAGE=unicef/etools-db:dev2 +VOLUME_ETOOLS_DATA?=/Users/Shared/data/storage/etools-db/data +VOLUME_ETOOLS_BACKUP?=/Users/Shared/data/storage/etools-db/dumps + build: docker build \ - -t unicef/etools-db:dev \ + -t ${DOCKER_IMAGE} \ -f Dockerfile . clean: - docker rmi unicef/etools-db:dev + docker rmi ${DOCKER_IMAGE} init: docker run --rm -it \ - -e DB_DUMP_LOCATION=/tmp/psql_data/db2.bz2 \ - -v /data/storage/etools-db/dumps:/tmp/psql_data/ \ - -v /data/storage/etools-db/data:/var/lib/postgresql/data \ - unicef/etools-db:dev + -e DB_DUMP_LOCATION=${DB_DUMP_LOCATION} \ + -v ${VOLUME_ETOOLS_BACKUP}:/tmp/psql_data/ \ + -v ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data \ + ${DOCKER_IMAGE} restore: docker run --rm -it \ - -e DB_DUMP_LOCATION=/tmp/psql_data/db2.bz2 \ - -v /data/storage/etools-db/dumps:/tmp/psql_data/ \ - -v /data/storage/etools-db/data:/var/lib/postgresql/data \ - unicef/etools-db:dev \ + -e DB_DUMP_LOCATION=${DB_DUMP_LOCATION} \ + -v ${VOLUME_ETOOLS_BACKUP}:/tmp/psql_data/ \ + -v ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data \ + ${DOCKER_IMAGE} \ /usr/local/bin/restore-db.sh run: docker run --rm \ -p 15432:5432 -it \ - -v /data/storage/etools-db/data:/var/lib/postgresql/data \ - unicef/etools-db:dev + -v ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data \ + ${DOCKER_IMAGE} + +stop: + docker stop ${DOCKER_IMAGE} shell: docker run --rm \ -it \ - -e DB_DUMP_LOCATION=/tmp/psql_data/db2.bz2 \ - -v /data/storage/etools-db/dumps:/tmp/psql_data/ \ - -v /data/storage/etools-db/data:/var/lib/postgresql/data \ - unicef/etools-db:dev \ + -e DB_DUMP_LOCATION=${DB_DUMP_LOCATION} \ + -v ${VOLUME_ETOOLS_BACKUP}:/tmp/psql_data/ \ + -v ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data \ + ${DOCKER_IMAGE} \ /bin/bash diff --git a/db/README.md b/db/README.md new file mode 100644 index 000000000..01cfd6c5d --- /dev/null +++ b/db/README.md @@ -0,0 +1,28 @@ +## Datamart eTools database + +This directory contains files and scripts needed to: + +- extract data from eTools backup +- create .sql files for test data +- builds docker image for the eTools test database + +- keep database updated to eTools schema + +**Read Makefile to check configuration variables. Consider your changes reading this file** + +### Build initial eTools test database : + +- get a .bz2 backup file from etools and name it `db2.bz2` +- copy somewhere and set `VOLUME_BACKUP` environment variable. + (it must point to the directory) +- run `make build init` + +### Run migrations: + +To keep test database updated without the need to rebuild it from a backup, you can run migrations +using a eTools docker image against the test database. +Simply run: + + - make run + - DATABASE_URL_ETOOLS=postgis://postgres:@127.0.0.1:5432/etools migrate-etools.sh` + - make stop diff --git a/db/load_db_data.sh b/db/load_db_data.sh index 92bc67ff0..7dc45f154 100755 --- a/db/load_db_data.sh +++ b/db/load_db_data.sh @@ -2,7 +2,7 @@ set -e - +echo "Loding db data" # Perform all actions as $POSTGRES_USER export PGUSER="$POSTGRES_USER" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..71d79b325 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,83 @@ +version: '2' +services: +# proxy: +# build: ./proxy +# ports: +# - 8000:8000 +# depends_on: +# - datamart +# - flower + +# base: +# build: +# context: . +# dockerfile: ./docker/Dockerfile.alpine.base +# args: +# VERSION: "x.x" + +# datamart: +# build: +# context: . +# dockerfile: ./docker/Dockerfile.alpine +# args: +# BASE_IMAGE: "2.11-base" +# environment: &environment +# AUTOCREATE_USERS: "admin,123" +# BASE_IMAGE: "compose" +# CACHE_URL: "redis://redis:6379/1" +# CACHE_URL_LOCK: "redis://redis:6379/1" +# CACHE_URL_API: "redis://redis:6379/1" +# CACHE_URL_TEMPLATE: "redis://redis:6379/1" +# CELERY_BROKER_URL: "redis://redis:6379/1" +# CELERY_RESULT_BACKEND: "redis://redis:6379/1" +## CELERY_BROKER_URL: "amqp://guest:guest@rabbitmq/datamart" +## CELERY_RESULT_BACKEND: "amqp://guest:guest@rabbitmq/datamart" +# CSRF_COOKIE_SECURE: 0 +# DATABASE_URL: postgis://postgres:@db:5432/etools_datamart +# DATABASE_URL_ETOOLS: postgis://postgres:@etools:5432/etools +# DEBUG: 0 +# SECRET_KEY: kuhfjhgfuytfuytfuygfuytdfuydfuygdfuygdfuytf +# SECURE_BROWSER_XSS_FILTER: 0 +# SECURE_CONTENT_TYPE_NOSNIFF: 0 +# SECURE_FRAME_DENY: 0 +# SECURE_HSTS_PRELOAD: 0 +# SECURE_HSTS_SECONDS: 0 +# SECURE_SSL_REDIRECT: 0 +# SESSION_COOKIE_HTTPONLY: 0 +# SESSION_COOKIE_SECURE: 0 +# STATIC_ROOT: /var/datamart/static/ +# STATIC_URL: /dm-static/ +# command: datamart +# ports: +# - 9999:8000 +# depends_on: +# - db +# - etools +# - redis + + db: + image: mdillon/postgis:9.6 + environment: + POSTGRES_PASSWORD: + POSTGRES_USER: postgres + POSTGRES_DB: etools_datamart + volumes: + - "$PWD/~build/db:/var/lib/postgresql/data" + ports: + - 25432:5432 + + etools: + image: mdillon/postgis:9.6 + ports: + - "15432:5432" + shm_size: '1gb' + environment: + POSTGRES_PASSWORD: + POSTGRES_USER: postgres + volumes: + - ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data + + + redis: + image: redis:alpine + diff --git a/docker/Makefile b/docker/Makefile index 4b5c2b4bd..81d424dd5 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=1 DOCKER_PASS?= DOCKER_USER?= -TARGET?=2.11 +TARGET?=2.11.1a PUSH_BASE?=1 BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" ) #BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" | echo "`xargs`xx" ) diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index d28a9926d..31289bde7 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,7 +1,7 @@ import warnings NAME = 'etools-datamart' -VERSION = __version__ = '2.11' +VERSION = __version__ = '2.12' __author__ = '' # UserWarning: The psycopg2 wheel package will be renamed from release 2.11; diff --git a/src/etools_datamart/apps/mart/data/migrations/0111_auto_20200221_1925.py b/src/etools_datamart/apps/mart/data/migrations/0111_auto_20200221_1925.py new file mode 100644 index 000000000..9b45f9b56 --- /dev/null +++ b/src/etools_datamart/apps/mart/data/migrations/0111_auto_20200221_1925.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.9 on 2020-02-21 19:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0110_auto_20191223_0126'), + ] + + operations = [ + migrations.AlterField( + model_name='intervention', + name='partner_signatory_title', + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name='interventionbudget', + name='partner_signatory_title', + field=models.CharField(max_length=100, null=True), + ), + migrations.AlterField( + model_name='interventionbylocation', + name='partner_signatory_title', + field=models.CharField(max_length=100, null=True), + ), + ] diff --git a/src/etools_datamart/apps/mart/data/models/intervention.py b/src/etools_datamart/apps/mart/data/models/intervention.py index 0dd87a7a4..bdbc902d3 100644 --- a/src/etools_datamart/apps/mart/data/models/intervention.py +++ b/src/etools_datamart/apps/mart/data/models/intervention.py @@ -71,7 +71,7 @@ class InterventionAbstract(models.Model): partner_signatory_first_name = models.CharField(max_length=64, null=True) partner_signatory_last_name = models.CharField(max_length=64, null=True) partner_signatory_phone = models.CharField(max_length=64, null=True) - partner_signatory_title = models.CharField(max_length=64, null=True) + partner_signatory_title = models.CharField(max_length=100, null=True) partner_source_id = models.IntegerField(blank=True, null=True) partner_type = models.CharField(max_length=64, null=True) partner_vendor_number = models.CharField(max_length=100, blank=True, null=True) diff --git a/src/etools_datamart/apps/mart/prp/migrations/0008_datareport_disaggregation.py b/src/etools_datamart/apps/mart/prp/migrations/0008_datareport_disaggregation.py new file mode 100644 index 000000000..1b4fdb2b1 --- /dev/null +++ b/src/etools_datamart/apps/mart/prp/migrations/0008_datareport_disaggregation.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.11 on 2020-04-02 14:17 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('prp', '0007_datareport_achievement_in_reporting_period'), + ] + + operations = [ + migrations.AddField( + model_name='datareport', + name='disaggregation', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True), + ), + ] diff --git a/src/etools_datamart/apps/mart/prp/models.py b/src/etools_datamart/apps/mart/prp/models.py index 2ed9d6472..8720bf7b7 100644 --- a/src/etools_datamart/apps/mart/prp/models.py +++ b/src/etools_datamart/apps/mart/prp/models.py @@ -45,6 +45,8 @@ Utilised budget(same conditions as above) Report # """ +from ast import literal_eval + from django.contrib.postgres.fields import JSONField from django.db import models, transaction from django.db.models import Q @@ -55,7 +57,7 @@ from etools_datamart.apps.etl.exceptions import MaxRecordsException, RequiredIsMissing, RequiredIsRunning from etools_datamart.apps.etl.loader import BaseLoader, EtlResult, logger, RUN_UNKNOWN -from etools_datamart.apps.sources.source_prp.models import (CoreCountry, CoreGatewaytype, +from etools_datamart.apps.sources.source_prp.models import (CoreCountry, CoreGatewaytype, IndicatorDisaggregationvalue, IndicatorIndicatorlocationdata, IndicatorIndicatorreport, IndicatorReportable, IndicatorReportablelocationgoal, UnicefLowerleveloutput, UnicefPdresultlink, @@ -302,6 +304,30 @@ def get_progress_report(self, record: IndicatorIndicatorlocationdata, values, ** pr.id, pd.title, pr.start_date, pr.end_date ) + def get_disaggregation(self, record: IndicatorIndicatorlocationdata, values, **kwargs): + # get disaggregation data and replace keys with disaggration value + disaggKeyValues = {} + for pk, value in IndicatorDisaggregationvalue.objects.values_list("pk", "value").all(): + try: + pk = int(pk) + except TypeError: + # probably a None, so we can ignore + pass + disaggKeyValues[pk] = value + disagg = {} + for k, v in record.disaggregation.items(): + k = literal_eval(k) + if len(k): + nk = [] + for i in k: + try: + nk.append(disaggKeyValues[int(i)]) + except KeyError: + nk.append(int(i)) + k = tuple(nk) + disagg[str(k)] = v + return disagg + class DataReport(PrpDataMartModel): # | indicator_report | idl.indicator_report | @@ -400,6 +426,7 @@ class DataReport(PrpDataMartModel): # total_cumulative_progress | reportable.total["c"] | total_cumulative_progress = models.CharField(max_length=2048, blank=True, null=True) achievement_in_reporting_period = models.CharField(max_length=2048, blank=True, null=True) + disaggregation = JSONField(blank=True, null=True) loader = DataReportLoader() @@ -450,5 +477,5 @@ class Options: 'total_cumulative_progress_in_location': 'N/A', 'total_cumulative_progress': '-', 'achievement_in_reporting_period': '-', - + 'disaggregation': '-', }