merge release/2.12

.bumpversion.cfg

@@ -27,3 +27,4 @@ values =
 
 [bumpversion:file:pyproject.toml]
 
+[bumpversion:file:docker-compose.yml]

.dockerignore

@@ -3,3 +3,4 @@
 db
 docs
 tests
+build

.gitignore

@@ -7,6 +7,7 @@
 .coverage
 .DS_Store
 .env
+.envrc
 .idea
 .pydevproject
 .tox
@@ -41,3 +42,4 @@ xml
 ~*
 **/dual-listbox-master/
 #src/etools_datamart/apps/security/static/security/dual-listbox-master
+build/

db/Makefile

@@ -1,39 +1,47 @@
+DB_DUMP_LOCATION?=/tmp/psql_data/db2.bz2
+DOCKER_IMAGE=unicef/etools-db:dev2
+VOLUME_ETOOLS_DATA?=/Users/Shared/data/storage/etools-db/data
+VOLUME_ETOOLS_BACKUP?=/Users/Shared/data/storage/etools-db/dumps
+
 build:
 	docker build \
-		-t unicef/etools-db:dev \
+		-t ${DOCKER_IMAGE} \
 		-f Dockerfile .
 
 
 clean:
-	docker rmi unicef/etools-db:dev
+	docker rmi ${DOCKER_IMAGE}
 
 init:
 	docker run --rm -it \
-			-e DB_DUMP_LOCATION=/tmp/psql_data/db2.bz2 \
-			-v /data/storage/etools-db/dumps:/tmp/psql_data/ \
-			-v /data/storage/etools-db/data:/var/lib/postgresql/data \
-			unicef/etools-db:dev
+			-e DB_DUMP_LOCATION=${DB_DUMP_LOCATION} \
+			-v ${VOLUME_ETOOLS_BACKUP}:/tmp/psql_data/ \
+			-v ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data \
+			${DOCKER_IMAGE}
 
 restore:
 	docker run --rm -it \
-			-e DB_DUMP_LOCATION=/tmp/psql_data/db2.bz2 \
-			-v /data/storage/etools-db/dumps:/tmp/psql_data/ \
-			-v /data/storage/etools-db/data:/var/lib/postgresql/data \
-			unicef/etools-db:dev \
+			-e DB_DUMP_LOCATION=${DB_DUMP_LOCATION} \
+			-v ${VOLUME_ETOOLS_BACKUP}:/tmp/psql_data/ \
+			-v ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data \
+			${DOCKER_IMAGE} \
 			/usr/local/bin/restore-db.sh
 
 run:
 	docker run --rm \
 		-p 15432:5432 -it \
-		-v /data/storage/etools-db/data:/var/lib/postgresql/data \
-		unicef/etools-db:dev
+		-v ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data \
+		${DOCKER_IMAGE}
+
+stop:
+	docker stop ${DOCKER_IMAGE}
 
 
 shell:
 	docker run --rm \
 		-it \
-		-e DB_DUMP_LOCATION=/tmp/psql_data/db2.bz2 \
-		-v /data/storage/etools-db/dumps:/tmp/psql_data/ \
-		-v /data/storage/etools-db/data:/var/lib/postgresql/data \
-		unicef/etools-db:dev \
+		-e DB_DUMP_LOCATION=${DB_DUMP_LOCATION} \
+		-v ${VOLUME_ETOOLS_BACKUP}:/tmp/psql_data/ \
+		-v ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data \
+		${DOCKER_IMAGE} \
 		/bin/bash

db/README.md

@@ -0,0 +1,28 @@
+## Datamart eTools database 
+
+This directory contains files and scripts needed to:
+
+- extract data from eTools backup
+- create .sql files for test data
+- builds docker image for the eTools test database
+
+- keep database updated to eTools schema
+
+**Read Makefile to check configuration variables. Consider your changes reading this file**
+
+### Build initial eTools test database :
+
+- get a .bz2 backup file from etools and name it `db2.bz2`
+- copy somewhere and set `VOLUME_BACKUP` environment variable.
+    (it must point to the directory)
+- run `make build init`
+
+### Run migrations:
+
+To keep test database updated without the need to rebuild it from a backup, you can run migrations
+using a eTools docker image against the test database.
+Simply run:
+
+    - make run
+    - DATABASE_URL_ETOOLS=postgis://postgres:@127.0.0.1:5432/etools migrate-etools.sh`
+    - make stop

db/load_db_data.sh

@@ -2,7 +2,7 @@
 
 
 set -e
-
+echo "Loding db data"
 # Perform all actions as $POSTGRES_USER
 export PGUSER="$POSTGRES_USER"
 

docker-compose.yml

@@ -0,0 +1,83 @@
+version: '2'
+services:
+#  proxy:
+#    build: ./proxy
+#    ports:
+#    - 8000:8000
+#    depends_on:
+#    - datamart
+#    - flower
+
+#  base:
+#    build:
+#      context: .
+#      dockerfile: ./docker/Dockerfile.alpine.base
+#      args:
+#        VERSION: "x.x"
+
+#  datamart:
+#    build:
+#      context: .
+#      dockerfile: ./docker/Dockerfile.alpine
+#      args:
+#        BASE_IMAGE: "2.11-base"
+#    environment: &environment
+#      AUTOCREATE_USERS: "admin,123"
+#      BASE_IMAGE: "compose"
+#      CACHE_URL: "redis://redis:6379/1"
+#      CACHE_URL_LOCK: "redis://redis:6379/1"
+#      CACHE_URL_API: "redis://redis:6379/1"
+#      CACHE_URL_TEMPLATE: "redis://redis:6379/1"
+#      CELERY_BROKER_URL: "redis://redis:6379/1"
+#      CELERY_RESULT_BACKEND: "redis://redis:6379/1"
+##      CELERY_BROKER_URL: "amqp://guest:guest@rabbitmq/datamart"
+##      CELERY_RESULT_BACKEND: "amqp://guest:guest@rabbitmq/datamart"
+#      CSRF_COOKIE_SECURE: 0
+#      DATABASE_URL: postgis://postgres:@db:5432/etools_datamart
+#      DATABASE_URL_ETOOLS: postgis://postgres:@etools:5432/etools
+#      DEBUG: 0
+#      SECRET_KEY: kuhfjhgfuytfuytfuygfuytdfuydfuygdfuygdfuytf
+#      SECURE_BROWSER_XSS_FILTER: 0
+#      SECURE_CONTENT_TYPE_NOSNIFF: 0
+#      SECURE_FRAME_DENY: 0
+#      SECURE_HSTS_PRELOAD: 0
+#      SECURE_HSTS_SECONDS: 0
+#      SECURE_SSL_REDIRECT: 0
+#      SESSION_COOKIE_HTTPONLY: 0
+#      SESSION_COOKIE_SECURE: 0
+#      STATIC_ROOT: /var/datamart/static/
+#      STATIC_URL: /dm-static/
+#    command: datamart
+#    ports:
+#      - 9999:8000
+#    depends_on:
+#      - db
+#      - etools
+#      - redis
+
+  db:
+    image: mdillon/postgis:9.6
+    environment:
+      POSTGRES_PASSWORD:
+      POSTGRES_USER: postgres
+      POSTGRES_DB: etools_datamart
+    volumes:
+      - "$PWD/~build/db:/var/lib/postgresql/data"
+    ports:
+      - 25432:5432
+
+  etools:
+    image: mdillon/postgis:9.6
+    ports:
+      - "15432:5432"
+    shm_size: '1gb'
+    environment:
+      POSTGRES_PASSWORD:
+      POSTGRES_USER: postgres
+    volumes:
+      - ${VOLUME_ETOOLS_DATA}:/var/lib/postgresql/data
+
+
+  redis:
+    image: redis:alpine
+

docker/Makefile

@@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?=
 DEVELOP?=1
 DOCKER_PASS?=
 DOCKER_USER?=
-TARGET?=2.11
+TARGET?=2.11.1a
 PUSH_BASE?=1
 BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" )
 #BASE?=$(shell echo "${TARGET}" | sed "s/\([0-9]*\)\.\([0-9]*\)\.\(.*\)/\1.\2/g" | echo "`xargs`xx" )

src/etools_datamart/__init__.py

@@ -1,7 +1,7 @@
 import warnings
 
 NAME = 'etools-datamart'
-VERSION = __version__ = '2.11'
+VERSION = __version__ = '2.12'
 __author__ = ''
 
 # UserWarning: The psycopg2 wheel package will be renamed from release 2.11;

src/etools_datamart/apps/mart/data/migrations/0111_auto_20200221_1925.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.2.9 on 2020-02-21 19:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data', '0110_auto_20191223_0126'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='intervention',
+            name='partner_signatory_title',
+            field=models.CharField(max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='interventionbudget',
+            name='partner_signatory_title',
+            field=models.CharField(max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='interventionbylocation',
+            name='partner_signatory_title',
+            field=models.CharField(max_length=100, null=True),
+        ),
+    ]

src/etools_datamart/apps/mart/data/models/intervention.py

@@ -71,7 +71,7 @@ class InterventionAbstract(models.Model):
     partner_signatory_first_name = models.CharField(max_length=64, null=True)
     partner_signatory_last_name = models.CharField(max_length=64, null=True)
     partner_signatory_phone = models.CharField(max_length=64, null=True)
-    partner_signatory_title = models.CharField(max_length=64, null=True)
+    partner_signatory_title = models.CharField(max_length=100, null=True)
     partner_source_id = models.IntegerField(blank=True, null=True)
     partner_type = models.CharField(max_length=64, null=True)
     partner_vendor_number = models.CharField(max_length=100, blank=True, null=True)

src/etools_datamart/apps/mart/prp/migrations/0008_datareport_disaggregation.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.2.11 on 2020-04-02 14:17
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('prp', '0007_datareport_achievement_in_reporting_period'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='datareport',
+            name='disaggregation',
+            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, null=True),
+        ),
+    ]

src/etools_datamart/apps/mart/prp/models.py

@@ -45,6 +45,8 @@
 Utilised budget(same conditions as above)
 Report  #
 """
+from ast import literal_eval
+
 from django.contrib.postgres.fields import JSONField
 from django.db import models, transaction
 from django.db.models import Q
@@ -55,7 +57,7 @@
 
 from etools_datamart.apps.etl.exceptions import MaxRecordsException, RequiredIsMissing, RequiredIsRunning
 from etools_datamart.apps.etl.loader import BaseLoader, EtlResult, logger, RUN_UNKNOWN
-from etools_datamart.apps.sources.source_prp.models import (CoreCountry, CoreGatewaytype,
+from etools_datamart.apps.sources.source_prp.models import (CoreCountry, CoreGatewaytype, IndicatorDisaggregationvalue,
                                                             IndicatorIndicatorlocationdata, IndicatorIndicatorreport,
                                                             IndicatorReportable, IndicatorReportablelocationgoal,
                                                             UnicefLowerleveloutput, UnicefPdresultlink,
@@ -302,6 +304,30 @@ def get_progress_report(self, record: IndicatorIndicatorlocationdata, values, **
                 pr.id, pd.title, pr.start_date, pr.end_date
             )
 
+    def get_disaggregation(self, record: IndicatorIndicatorlocationdata, values, **kwargs):
+        # get disaggregation data and replace keys with disaggration value
+        disaggKeyValues = {}
+        for pk, value in IndicatorDisaggregationvalue.objects.values_list("pk", "value").all():
+            try:
+                pk = int(pk)
+            except TypeError:
+                # probably a None, so we can ignore
+                pass
+            disaggKeyValues[pk] = value
+        disagg = {}
+        for k, v in record.disaggregation.items():
+            k = literal_eval(k)
+            if len(k):
+                nk = []
+                for i in k:
+                    try:
+                        nk.append(disaggKeyValues[int(i)])
+                    except KeyError:
+                        nk.append(int(i))
+                k = tuple(nk)
+            disagg[str(k)] = v
+        return disagg
+
 
 class DataReport(PrpDataMartModel):
     # | indicator_report | idl.indicator_report |
@@ -400,6 +426,7 @@ class DataReport(PrpDataMartModel):
     # total_cumulative_progress | reportable.total["c"] |
     total_cumulative_progress = models.CharField(max_length=2048, blank=True, null=True)
     achievement_in_reporting_period = models.CharField(max_length=2048, blank=True, null=True)
+    disaggregation = JSONField(blank=True, null=True)
 
     loader = DataReportLoader()
 
@@ -450,5 +477,5 @@ class Options:
                    'total_cumulative_progress_in_location': 'N/A',
                    'total_cumulative_progress': '-',
                    'achievement_in_reporting_period': '-',
-
+                   'disaggregation': '-',
                    }