From 3483e2b92fe1b32ef5cb69e9607d3a1e9c0b849b Mon Sep 17 00:00:00 2001 From: prabinoid Date: Tue, 3 Sep 2024 14:25:03 +0545 Subject: [PATCH 1/3] fix: Campaigns list and allowed usernames list --- backend/api/projects/resources.py | 1 + backend/models/postgis/project.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/api/projects/resources.py b/backend/api/projects/resources.py index 9c07b4b788..48294149f1 100644 --- a/backend/api/projects/resources.py +++ b/backend/api/projects/resources.py @@ -915,6 +915,7 @@ async def get(request: Request, project_id: int, db: Database = Depends(get_db)) description: Internal Server Error """ preferred_locale = request.headers.get("accept-language") + print(request.user.display_name) summary = await ProjectService.get_project_summary(project_id, db, preferred_locale) return summary diff --git a/backend/models/postgis/project.py b/backend/models/postgis/project.py index a5c976f180..99fafd3484 100644 --- a/backend/models/postgis/project.py +++ b/backend/models/postgis/project.py @@ -961,11 +961,12 @@ async def get_project_summary( if summary.private: allowed_user_ids = project_row.allowed_users if project_row.allowed_users else [] if allowed_user_ids: - query = "SELECT username FROM users WHERE id IN :allowed_user_ids" + query = "SELECT username FROM users WHERE id = ANY(:allowed_user_ids)" allowed_users = await db.fetch_all(query, {"allowed_user_ids": allowed_user_ids}) summary.allowed_users = [user["username"] for user in allowed_users] else: summary.allowed_users = [] + # AOI centroid summary.aoi_centroid = geojson.loads(project_row.centroid) @@ -984,7 +985,8 @@ async def get_project_summary( """ campaigns = await db.fetch_all(query=query, values={"project_id": project_id}) - summary.campaigns = Campaign.campaign_list_as_dto(campaigns) + campaigns_dto = [CampaignDTO(**campaign) for campaign in campaigns] if campaigns else [] + summary.campaigns = campaigns_dto # Project teams query = """ From e80c40926fab7c38e63e9263eca38073276af72e Mon Sep 17 00:00:00 2001 From: prabinoid Date: Tue, 3 Sep 2024 17:54:28 +0545 Subject: [PATCH 2/3] Project user activities apis refactored --- backend/api/projects/activities.py | 23 ++-- backend/api/projects/resources.py | 1 - backend/api/projects/statistics.py | 2 - backend/models/dtos/mapping_dto.py | 14 +- backend/models/dtos/stats_dto.py | 12 +- backend/services/stats_service.py | 203 ++++++++++++++++++----------- 6 files changed, 146 insertions(+), 109 deletions(-) diff --git a/backend/api/projects/activities.py b/backend/api/projects/activities.py index 4545c264b9..50dbc8a594 100644 --- a/backend/api/projects/activities.py +++ b/backend/api/projects/activities.py @@ -1,8 +1,9 @@ from fastapi import APIRouter, Depends, Request from backend.services.stats_service import StatsService from backend.services.project_service import ProjectService -from backend.db import get_session - +from backend.db import get_db, get_session +from databases import Database +from fastapi import HTTPException router = APIRouter( @@ -12,9 +13,8 @@ responses={404: {"description": "Not found"}}, ) -# class ProjectsActivitiesAPI(Resource): @router.get("/{project_id}/activities/") -async def get(request: Request, project_id): +async def get(request: Request, project_id: int, db: Database = Depends(get_db)): """ Get all user activity on a project --- @@ -41,15 +41,14 @@ async def get(request: Request, project_id): 500: description: Internal Server Error """ - ProjectService.exists(project_id) + await ProjectService.exists(project_id, db) page = int(request.query_params.get("page")) if request.query_params.get("page") else 1 - activity = StatsService.get_latest_activity(project_id, page) - return activity.model_dump(by_alias=True), 200 + activity = await StatsService.get_latest_activity(project_id, page, db) + return activity -# class ProjectsLastActivitiesAPI(Resource): @router.get("/{project_id}/activities/latest/") -async def get(request: Request, project_id): +async def get(request: Request, project_id: int, db: Database = Depends(get_db)): """ Get latest user activity on all of project task --- @@ -71,6 +70,6 @@ async def get(request: Request, project_id): 500: description: Internal Server Error """ - ProjectService.exists(project_id) - activity = StatsService.get_last_activity(project_id) - return activity.model_dump(by_alias=True), 200 + await ProjectService.exists(project_id, db) + activity = await StatsService.get_last_activity(project_id, db) + return activity diff --git a/backend/api/projects/resources.py b/backend/api/projects/resources.py index 48294149f1..9c07b4b788 100644 --- a/backend/api/projects/resources.py +++ b/backend/api/projects/resources.py @@ -915,7 +915,6 @@ async def get(request: Request, project_id: int, db: Database = Depends(get_db)) description: Internal Server Error """ preferred_locale = request.headers.get("accept-language") - print(request.user.display_name) summary = await ProjectService.get_project_summary(project_id, db, preferred_locale) return summary diff --git a/backend/api/projects/statistics.py b/backend/api/projects/statistics.py index 4040c07405..575f62df9a 100644 --- a/backend/api/projects/statistics.py +++ b/backend/api/projects/statistics.py @@ -12,7 +12,6 @@ responses={404: {"description": "Not found"}}, ) -# class ProjectsStatisticsQueriesPopularAPI(Resource): @router.get("/queries/popular/") async def get(): """ @@ -32,7 +31,6 @@ async def get(): return stats.model_dump(by_alias=True), 200 -# class ProjectsStatisticsAPI(Resource): @router.get("/{project_id}/statistics/") async def get(project_id: int, session: AsyncSession = Depends(get_session)): """ diff --git a/backend/models/dtos/mapping_dto.py b/backend/models/dtos/mapping_dto.py index 2aa36b0754..9086a8f47d 100644 --- a/backend/models/dtos/mapping_dto.py +++ b/backend/models/dtos/mapping_dto.py @@ -53,7 +53,7 @@ class TaskHistoryDTO(BaseModel): """Describes an individual action that was performed on a mapping task""" history_id : Optional[int] = Field(alias="historyId", default=None) - task_id: Optional[str] = Field(alias="taskId", default=None) + task_id: Optional[int] = Field(alias="taskId", default=None) action: Optional[str] = None action_text: Optional[str] = Field(alias="actionText", default=None) action_date: datetime = Field(alias="actionDate", default=None) @@ -61,21 +61,21 @@ class TaskHistoryDTO(BaseModel): picture_url: Optional[str] = Field(alias="pictureUrl", default=None) issues: Optional[List[TaskMappingIssueDTO]] = None + class Config: + populate_by_name = True + class TaskStatusDTO(BaseModel): """Describes a DTO for the current status of the task""" - def __init__(self, task_status, **data): - super().__init__(**data) - self.task_id = task_status["task_id"] - self.task_status = task_status["task_status"] - self.action_date = task_status["action_date"] - self.action_by = task_status["action_by"] task_id: Optional[int] = Field(alias="taskId", default=None) task_status: Optional[str] = Field(alias="taskStatus", default=None) action_date: Optional[datetime] = Field(alias="actionDate", default=None) action_by: Optional[str] = Field(alias="actionBy", default=None) + class Config: + populate_by_name = True + class TaskDTO(BaseModel): """Describes a Task DTO""" diff --git a/backend/models/dtos/stats_dto.py b/backend/models/dtos/stats_dto.py index 2c2c3a0e89..b10482e79b 100644 --- a/backend/models/dtos/stats_dto.py +++ b/backend/models/dtos/stats_dto.py @@ -74,14 +74,8 @@ def from_total_count(page: int, per_page: int, total: int) -> "Pagination": total=total, ) - class ProjectActivityDTO(BaseModel): """DTO to hold all project activity""" - - def __init__(self): - super().__init__() - self.activity = [] - pagination: Optional[Pagination] = None activity: Optional[List[TaskHistoryDTO]] = None @@ -89,11 +83,7 @@ def __init__(self): class ProjectLastActivityDTO(BaseModel): """DTO to hold latest status from project activity""" - def __init__(self): - super().__init__() - self.activity = [] - - activity: Optional[List[TaskStatusDTO]] = None + activity: Optional[List[TaskStatusDTO]] = Field(default_factory=list) class OrganizationProjectsStatsDTO(BaseModel): draft: Optional[int] = None diff --git a/backend/services/stats_service.py b/backend/services/stats_service.py index 2c53f39b08..bf47deb43d 100644 --- a/backend/services/stats_service.py +++ b/backend/services/stats_service.py @@ -1,3 +1,4 @@ +from typing import Optional from cachetools import TTLCache, cached from datetime import date, timedelta import sqlalchemy as sa @@ -38,6 +39,8 @@ from backend.services.campaign_service import CampaignService from backend.db import get_session session = get_session() +from databases import Database +from fastapi import HTTPException homepage_stats_cache = TTLCache(maxsize=4, ttl=30) @@ -119,42 +122,73 @@ def _update_tasks_stats( return project, user @staticmethod - def get_latest_activity(project_id: int, page: int) -> ProjectActivityDTO: + async def get_latest_activity(project_id: int, page: int, db: Database) -> ProjectActivityDTO: """Gets all the activity on a project""" - - if not ProjectService.exists(project_id): - raise NotFound(sub_code="PROJECT_NOT_FOUND", project_id=project_id) - - results = ( - session.query( - TaskHistory.id, - TaskHistory.task_id, - TaskHistory.action, - TaskHistory.action_date, - TaskHistory.action_text, - User.username, - ) - .join(User) - .filter( - TaskHistory.project_id == project_id, - TaskHistory.action != TaskAction.COMMENT.name, + + # Pagination setup + page_size = 10 + offset = (page - 1) * page_size + + # Query to fetch task history + query = """ + SELECT + th.id, + th.task_id, + th.action, + th.action_date, + th.action_text, + u.username + FROM task_history th + JOIN users u ON th.user_id = u.id + WHERE + th.project_id = :project_id + AND th.action != :comment_action + ORDER BY th.action_date DESC + LIMIT :limit OFFSET :offset + """ + rows = await db.fetch_all(query, { + "project_id": project_id, + "comment_action": "COMMENT", + "limit": page_size, + "offset": offset + }) + + # Creating DTO + activity_dto = ProjectActivityDTO(activity=[]) + for row in rows: + history = TaskHistoryDTO( + history_id=row["id"], + task_id=row["task_id"], + action=row["action"], + action_text=row["action_text"], + action_date=row["action_date"], + action_by=row["username"] ) - .order_by(TaskHistory.action_date.desc()) - .paginate(page=page, per_page=10, error_out=True) - ) - - activity_dto = ProjectActivityDTO() - for item in results.items: - history = TaskHistoryDTO() - history.task_id = item.id - history.task_id = item.task_id - history.action = item.action - history.action_text = item.action_text - history.action_date = item.action_date - history.action_by = item.username activity_dto.activity.append(history) - activity_dto.pagination = Pagination(results) + # Calculate total items for pagination + total_query = """ + SELECT COUNT(*) + FROM task_history th + WHERE + th.project_id = :project_id + AND th.action != :comment_action + """ + total_items_result = await db.fetch_one(total_query, { + "project_id": project_id, + "comment_action": "COMMENT" + }) + + + total_items = total_items_result["count"] if total_items_result else 0 + + # Use the from_total_count method to correctly initialize the Pagination DTO + activity_dto.pagination = Pagination.from_total_count( + page=page, + per_page=page_size, + total=total_items + ) + return activity_dto @staticmethod @@ -198,54 +232,71 @@ def get_popular_projects() -> ProjectSearchResultsDTO: return dto @staticmethod - def get_last_activity(project_id: int) -> ProjectLastActivityDTO: + async def get_last_activity(project_id: int, db: Database) -> ProjectLastActivityDTO: """Gets the last activity for a project's tasks""" - sq = ( - session.query(TaskHistory).with_entities( - TaskHistory.task_id, - TaskHistory.action_date, - TaskHistory.user_id, - ) - .filter(TaskHistory.project_id == project_id) - .filter(TaskHistory.action != TaskAction.COMMENT.name) - .order_by(TaskHistory.task_id, TaskHistory.action_date.desc()) - .distinct(TaskHistory.task_id) - .subquery() - ) - - sq_statuses = ( - session.query(Task).with_entities(Task.id, Task.task_status) - .filter(Task.project_id == project_id) - .subquery() - ) - results = ( - session.query( - sq_statuses.c.id, - sq.c.action_date, - sq_statuses.c.task_status, - User.username, - ) - .outerjoin(sq, sq.c.task_id == sq_statuses.c.id) - .outerjoin(User, User.id == sq.c.user_id) - .order_by(sq_statuses.c.id) - .all() - ) - - dto = ProjectLastActivityDTO() - dto.activity = [ - TaskStatusDTO( - dict( - task_id=r.id, - task_status=TaskStatus(r.task_status).name, - action_date=r.action_date, - action_by=r.username, - ) - ) - for r in results - ] + + # First subquery: Fetch the latest action date per task, excluding comments + subquery_latest_action = """ + SELECT DISTINCT ON (th.task_id) + th.task_id, + th.action_date, + th.user_id + FROM task_history th + WHERE th.project_id = :project_id + AND th.action != :comment_action + ORDER BY th.task_id, th.action_date DESC + """ + + latest_actions = await db.fetch_all(subquery_latest_action, { + "project_id": project_id, + "comment_action": "COMMENT" + }) + + # Mapping the latest actions by task_id + latest_actions_map = {item["task_id"]: item for item in latest_actions} + + # Second subquery: Fetch the task statuses + query_task_statuses = """ + SELECT + t.id as task_id, + t.task_status, + u.username, + la.action_date + FROM tasks t + LEFT JOIN ( + SELECT + th.task_id, + th.action_date, + th.user_id + FROM task_history th + WHERE th.project_id = :project_id + AND th.action != :comment_action + ORDER BY th.task_id, th.action_date DESC + ) la ON la.task_id = t.id + LEFT JOIN users u ON u.id = la.user_id + WHERE t.project_id = :project_id + ORDER BY t.id + """ + + results = await db.fetch_all(query_task_statuses, { + "project_id": project_id, + "comment_action": "COMMENT" + }) + + # Creating DTO + dto = ProjectLastActivityDTO(activity=[]) + for row in results: + task_status_dto = TaskStatusDTO( + task_id=row["task_id"], + task_status=TaskStatus(row["task_status"]).name, + action_date=row["action_date"], + action_by=row["username"] + ) + dto.activity.append(task_status_dto) return dto + @staticmethod def get_user_contributions(project_id: int) -> ProjectContributionsDTO: """Get all user contributions on a project""" From 253761cb5a6ffb6ea41a76e920a1ce8f3e2b1757 Mon Sep 17 00:00:00 2001 From: prabinoid Date: Wed, 4 Sep 2024 17:18:06 +0545 Subject: [PATCH 3/3] Project contributions apis and other geom queries apis refactored --- backend/api/projects/contributions.py | 19 +-- backend/api/projects/resources.py | 40 +++-- backend/models/dtos/project_dto.py | 151 ++++++++++++------ backend/models/dtos/stats_dto.py | 20 +++ backend/models/postgis/project.py | 35 +++-- backend/services/project_admin_service.py | 6 +- backend/services/project_service.py | 104 ++++++------- backend/services/stats_service.py | 182 ++++++++++------------ 8 files changed, 304 insertions(+), 253 deletions(-) diff --git a/backend/api/projects/contributions.py b/backend/api/projects/contributions.py index 6500360045..91e737f780 100644 --- a/backend/api/projects/contributions.py +++ b/backend/api/projects/contributions.py @@ -1,9 +1,12 @@ # from flask_restful import Resource +from backend.models.postgis.project import Project from backend.services.project_service import ProjectService from backend.services.stats_service import StatsService from fastapi import APIRouter, Depends from backend.db import get_session +from backend.db import get_db +from databases import Database router = APIRouter( prefix="/projects", @@ -12,9 +15,8 @@ responses={404: {"description": "Not found"}}, ) -# class ProjectsContributionsAPI(Resource): @router.get("/{project_id}/contributions/") -async def get(project_id): +async def get(project_id: int, db: Database = Depends(get_db)): """ Get all user contributions on a project --- @@ -37,14 +39,13 @@ async def get(project_id): 500: description: Internal Server Error """ - ProjectService.exists(project_id) - contributions = StatsService.get_user_contributions(project_id) - return contributions.model_dump(by_alias=True), 200 + await Project.exists(project_id, db) + contributions = await StatsService.get_user_contributions(project_id, db) + return contributions -# class ProjectsContributionsQueriesDayAPI(Resource): @router.get("/{project_id}/contributions/queries/day/") -async def get(project_id): +async def get(project_id: int, db: Database = Depends(get_db)): """ Get contributions by day for a project --- @@ -67,5 +68,5 @@ async def get(project_id): 500: description: Internal Server Error """ - contribs = ProjectService.get_contribs_by_day(project_id) - return contribs.model_dump(by_alias=True), 200 + contribs = await ProjectService.get_contribs_by_day(project_id, db) + return contribs diff --git a/backend/api/projects/resources.py b/backend/api/projects/resources.py index 9c07b4b788..d57040b378 100644 --- a/backend/api/projects/resources.py +++ b/backend/api/projects/resources.py @@ -36,6 +36,7 @@ from starlette.authentication import requires import json from sqlalchemy.ext.asyncio import AsyncSession + from backend.db import get_db from databases import Database from backend.services.users.authentication_service import login_required @@ -920,7 +921,7 @@ async def get(request: Request, project_id: int, db: Database = Depends(get_db)) @router.get("/{project_id}/queries/nogeometries/") -async def get(request: Request, project_id): +async def get(request: Request, project_id: int, db: Database = Depends(get_db)): """ Get HOT Project for mapping --- @@ -963,11 +964,10 @@ async def get(request: Request, project_id): else False ) locale = request.headers.get("accept-language") - project_dto = ProjectService.get_project_dto_for_mapper( - project_id, None, locale, True + project_dto = await ProjectService.get_project_dto_for_mapper( + project_id, None, db, locale, True ) - project_dto = project_dto.model_dump(by_alias=True) - + #TODO Send file. if as_file: return send_file( io.BytesIO(geojson.dumps(project_dto).encode("utf-8")), @@ -976,7 +976,7 @@ async def get(request: Request, project_id): download_name=f"project_{str(project_id)}.json", ) - return project_dto, 200 + return project_dto except ProjectServiceError as e: return {"Error": str(e).split("-")[1], "SubCode": str(e).split("-")[0]}, 403 finally: @@ -988,8 +988,7 @@ async def get(request: Request, project_id): @router.get("/{project_id}/queries/notasks/") -@requires("authenticated") -async def get(request: Request, project_id): +async def get(request: Request, project_id: int, db: Database = Depends(get_db), user: AuthUserDTO = Depends(login_required)): """ Retrieves a Tasking-Manager project --- @@ -1022,21 +1021,20 @@ async def get(request: Request, project_id): 500: description: Internal Server Error """ - if not ProjectAdminService.is_user_action_permitted_on_project( - request.user.display_name, project_id + if not await ProjectAdminService.is_user_action_permitted_on_project( + request.user.display_name, project_id, db ): return { "Error": "User is not a manager of the project", "SubCode": "UserPermissionError", }, 403 - project_dto = ProjectAdminService.get_project_dto_for_admin(project_id) - return project_dto.model_dump(by_alias=True), 200 + project_dto = await ProjectAdminService.get_project_dto_for_admin(project_id, db) + return project_dto -# class ProjectsQueriesAoiAPI(): @router.get("/{project_id}/queries/aoi/") -async def get(request: Request, project_id): +async def get(request: Request, project_id: int, db: Database = Depends(get_db)): """ Get AOI of Project --- @@ -1069,11 +1067,11 @@ async def get(request: Request, project_id): as_file = ( strtobool(request.query_params.get("as_file")) if request.query_params.get("as_file") - else True + else False ) - project_aoi = ProjectService.get_project_aoi(project_id) - + project_aoi = await ProjectService.get_project_aoi(project_id, db) + #TODO as file. if as_file: return send_file( io.BytesIO(geojson.dumps(project_aoi).encode("utf-8")), @@ -1082,11 +1080,11 @@ async def get(request: Request, project_id): download_name=f"{str(project_id)}.geojson", ) - return project_aoi, 200 + return project_aoi @router.get("/{project_id}/queries/priority-areas/") -async def get(project_id): +async def get(project_id: int, db: Database = Depends(get_db)): """ Get Priority Areas of a project --- @@ -1112,8 +1110,8 @@ async def get(project_id): description: Internal Server Error """ try: - priority_areas = ProjectService.get_project_priority_areas(project_id) - return priority_areas, 200 + priority_areas = await ProjectService.get_project_priority_areas(project_id, db) + return priority_areas except ProjectServiceError: return {"Error": "Unable to fetch project"}, 403 diff --git a/backend/models/dtos/project_dto.py b/backend/models/dtos/project_dto.py index 428a102174..4417d0fd90 100644 --- a/backend/models/dtos/project_dto.py +++ b/backend/models/dtos/project_dto.py @@ -15,10 +15,10 @@ ProjectDifficulty, ) from backend.models.dtos.campaign_dto import CampaignDTO - from pydantic import BaseModel, Field, ValidationError, field_validator -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union from datetime import datetime +from datetime import date from typing_extensions import Annotated from fastapi import HTTPException @@ -190,52 +190,104 @@ class CustomEditorDTO(BaseModel): class ProjectDTO(BaseModel): """Describes JSON model for a tasking manager project""" - - project_id: Optional[int] = Field(serialization_alias="projectId", default=None) - project_status: Optional[str] = None - project_priority: Optional[str] = None - area_of_interest: Optional[dict] = Field(serialization_alias="areaOfInterest", default={}) - aoi_bbox: List[float] = Field(serialization_alias="aoiBBOX", default=[]) - tasks: Optional[dict] = None # Replace with the actual type for tasks - default_locale: Optional[str] = Field(serialization_alias="defaultLocale", default=None) - project_info: Optional[ProjectInfoDTO] = None - project_info_locales: List[ProjectInfoDTO] = Field(serialization_alias="projectInfoLocales", default=[]) - difficulty: Optional[str] = None - mapping_permission: Optional[str] = Field(serialization_alias="mappingPermission", default=None) - validation_permission: Optional[str] = Field(serialization_alias="validationPermission", default=None) - enforce_random_task_selection: Optional[bool] = Field(serialization_alias="enforceRandomTaskSelection", default=False) - private: Optional[bool] = None - changeset_comment: Optional[str] = Field(serialization_alias="changesetComment", default=None) - osmcha_filter_id: Optional[str] = Field(serialization_alias="osmchaFilterId", default=None) - due_date: Optional[str] = Field(serialization_alias="dueDate", default=None) + + project_id: Optional[int] = Field(None, alias="projectId") + project_status: str = Field(alias="status") + project_priority: str = Field(alias="projectPriority") + area_of_interest: Optional[dict] = Field(None, alias="areaOfInterest") + aoi_bbox: Optional[List[float]] = Field(None, alias="aoiBBOX") + tasks: Optional[dict] = None + default_locale: str = Field(alias="defaultLocale") + project_info: Optional[ProjectInfoDTO] = Field(None, alias="projectInfo") + project_info_locales: Optional[List["ProjectInfoDTO"]] = Field(None, alias="projectInfoLocales") + difficulty: str = Field(alias="difficulty") + mapping_permission: str = Field(alias="mappingPermission") + validation_permission: str = Field(alias="validationPermission") + enforce_random_task_selection: Optional[bool] = Field(False, alias="enforceRandomTaskSelection") + private: bool + changeset_comment: Optional[str] = Field(None, alias="changesetComment") + osmcha_filter_id: Optional[str] = Field(None, alias="osmchaFilterId") + due_date: Optional[datetime] = Field(None, alias="dueDate") imagery: Optional[str] = None - josm_preset: Optional[str] = Field(serialization_alias="josmPreset", default=None) - id_presets: Optional[List[str]] = Field(serialization_alias="idPresets", default=[]) - extra_id_params: Optional[str] = Field(serialization_alias="extraIdParams", default=None) - rapid_power_user: Optional[bool] = Field(serialization_alias="rapidPowerUser", default=False) - mapping_types: List[str] = Field(serialization_alias="mappingTypes", default=[], validators=[is_known_mapping_type]) + josm_preset: Optional[str] = Field(None, alias="josmPreset") + id_presets: Optional[List[str]] = Field(default=[], alias="idPresets") + extra_id_params: Optional[str] = Field(None, alias="extraIdParams") + rapid_power_user: Optional[bool] = Field(False, alias="rapidPowerUser") + mapping_types: List[str] = Field(default=[], alias="mappingTypes") campaigns: List[CampaignDTO] = Field(default=[]) - organisation: Optional[int] = None - organisation_name: Optional[str] = Field(serialization_alias="organisationName", default=None) - organisation_slug: Optional[str] = Field(serialization_alias="organisationSlug", default=None) - organisation_logo: Optional[str] = Field(serialization_alias="organisationLogo", default=None) - country_tag: Optional[List[str]] = Field(serialization_alias="countryTag", default=[]) - license_id: Optional[int] = Field(serialization_alias="licenseId", default=None) - allowed_usernames: List[str] = Field(serialization_alias="allowedUsernames", default=[]) - priority_areas: Optional[dict] = Field(serialization_alias="priorityAreas", default={}) - created: Optional[str] = None - last_updated: Optional[str] = Field(serialization_alias="lastUpdated", default=None) + organisation: int + organisation_name: Optional[str] = Field(None, alias="organisationName") + organisation_slug: Optional[str] = Field(None, alias="organisationSlug") + organisation_logo: Optional[str] = Field(None, alias="organisationLogo") + country_tag: Optional[List[str]] = Field(None, alias="countryTag") + license_id: Optional[int] = Field(None, alias="licenseId") + allowed_usernames: Optional[List[str]] = Field(default=[], alias="allowedUsernames") + priority_areas: Optional[Dict] = Field(None, alias="priorityAreas") + created: Optional[datetime] = None + last_updated: Optional[datetime] = Field(None, alias="lastUpdated") author: Optional[str] = None - active_mappers: Optional[int] = Field(serialization_alias="activeMappers", default=None) - percent_mapped: Optional[int] = Field(serialization_alias="percentMapped", default=None) - percent_validated: Optional[int] = Field(serialization_alias="percentValidated", default=None) - percent_bad_imagery: Optional[int] = Field(serialization_alias="percentBadImagery", default=None) - task_creation_mode: Optional[str] = Field(serialization_alias="taskCreationMode", validators=[is_known_task_creation_mode], default=None) - project_teams: List[ProjectTeamDTO] = Field(serialization_alias="teams", default=[]) - mapping_editors: Optional[List[str]] = Field(serialization_alias="mappingEditors", min_items=1, validators=[is_known_editor], default=[]) - validation_editors: Optional[List[str]] = Field(serialization_alias="validationEditors", min_items=1, validators=[is_known_editor], default=[]) - custom_editor: Optional[CustomEditorDTO] = Field(serialization_alias="customEditor", default=None) - interests: Optional[List[ListInterestDTO]] = None + active_mappers: Optional[int] = Field(None, alias="activeMappers") + percent_mapped: Optional[int] = Field(None, alias="percentMapped") + percent_validated: Optional[int] = Field(None, alias="percentValidated") + percent_bad_imagery: Optional[int] = Field(None, alias="percentBadImagery") + task_creation_mode: str = Field(alias="taskCreationMode") + project_teams: Optional[List[ProjectTeamDTO]] = Field(None, alias="teams") + mapping_editors: List[str] = Field(alias="mappingEditors") + validation_editors: List[str] = Field(alias="validationEditors") + custom_editor: Optional[CustomEditorDTO] = Field(None, alias="customEditor") + interests: Optional[List[InterestDTO]] = None + + class Config: + populate_by_name = True + + #TODO CHeck validators. + # @validator('project_status') + # def validate_project_status(cls, value): + # if not is_known_project_status(value): + # raise ValueError('Invalid project status') + # return value + + # @validator('project_priority') + # def validate_project_priority(cls, value): + # if not is_known_project_priority(value): + # raise ValueError('Invalid project priority') + # return value + + # @validator('difficulty') + # def validate_difficulty(cls, value): + # if not is_known_project_difficulty(value): + # raise ValueError('Invalid project difficulty') + # return value + + # @validator('mapping_permission') + # def validate_mapping_permission(cls, value): + # if not is_known_mapping_permission(value): + # raise ValueError('Invalid mapping permission') + # return value + + # @validator('validation_permission') + # def validate_validation_permission(cls, value): + # if not is_known_validation_permission(value): + # raise ValueError('Invalid validation permission') + # return value + + # @validator('mapping_types', each_item=True) + # def validate_mapping_types(cls, value): + # if not is_known_mapping_type(value): + # raise ValueError('Invalid mapping type') + # return value + + # @validator('task_creation_mode') + # def validate_task_creation_mode(cls, value): + # if not is_known_task_creation_mode(value): + # raise ValueError('Invalid task creation mode') + # return value + + # @validator('mapping_editors', 'validation_editors', each_item=True) + # def validate_editors(cls, value): + # if not is_known_editor(value): + # raise ValueError('Invalid editor') + # return value class ProjectFavoriteDTO(BaseModel): @@ -391,14 +443,13 @@ def __init__(self): comments: List[ProjectComment] - class ProjectContribDTO(BaseModel): - date: datetime + date: date mapped: int validated: int - cumulative_mapped: Optional[int] - cumulative_validated: Optional[int] - total_tasks: Optional[int] + cumulative_mapped: Optional[int] = None + cumulative_validated: Optional[int] = None + total_tasks: Optional[int] = None class ProjectContribsDTO(BaseModel): diff --git a/backend/models/dtos/stats_dto.py b/backend/models/dtos/stats_dto.py index b10482e79b..89d74c0d25 100644 --- a/backend/models/dtos/stats_dto.py +++ b/backend/models/dtos/stats_dto.py @@ -36,6 +36,26 @@ def __init__(self, UserContribution): date_registered: Optional[datetime] = Field(alias="dateRegistered", default=None) +# class UserContribution(BaseModel): +# """User contribution for a project""" + +# username: Optional[str] = None +# mapping_level: Optional[str] = Field(None, alias="mappingLevel") +# picture_url: Optional[str] = Field(None, alias="pictureUrl") +# mapped: Optional[int] = None +# validated: Optional[int] = None +# bad_imagery: Optional[int] = Field(None, alias="badImagery") +# total: Optional[int] = None +# mapped_tasks: Optional[List[int]] = Field(default_factory=list, alias="mappedTasks") +# validated_tasks: Optional[List[int]] = Field(default_factory=list, alias="validatedTasks") +# bad_imagery_tasks: Optional[List[int]] = Field(default_factory=list, alias="badImageryTasks") +# name: Optional[str] = None +# date_registered: Optional[datetime] = Field(None, alias="dateRegistered") + +# class Config: +# allow_population_by_field_name = True + + class ProjectContributionsDTO(BaseModel): """DTO for all user contributions on a project""" diff --git a/backend/models/postgis/project.py b/backend/models/postgis/project.py index 99fafd3484..f23393dffa 100644 --- a/backend/models/postgis/project.py +++ b/backend/models/postgis/project.py @@ -546,12 +546,20 @@ def delete(self): session.commit() @staticmethod - async def exists(project_id: int, session): - query = select(literal(True)).where( - select(Project.id).filter(Project.id == project_id).exists() - ) - result = await session.execute(query) - return result.scalar() + async def exists(project_id: int, db: Database) -> bool: + query = """ + SELECT 1 + FROM projects + WHERE id = :project_id + """ + + # Execute the query + result = await db.fetch_one(query=query, values={"project_id": project_id}) + + if result is None: + raise NotFound(sub_code="PROJECT_NOT_FOUND", project_id=project_id) + + return True def is_favorited(self, user_id: int) -> bool: user = session.get(User, user_id) @@ -1374,16 +1382,15 @@ def calculate_tasks_percent( ) except ZeroDivisionError: return 0 + - def as_dto_for_admin(self, project_id): + @staticmethod + async def as_dto_for_admin(project_id: int, db: Database ): """Creates a Project DTO suitable for transmitting to project admins""" - project, project_dto = self.get_project_and_base_dto() - - if project is None: - return None - - project_dto.project_info_locales = ProjectInfo.get_dto_for_all_locales( - project_id + project_dto = await Project.get_project_and_base_dto(project_id, db) + + project_dto.project_info_locales = await ProjectInfo.get_dto_for_all_locales( + db, project_id ) return project_dto diff --git a/backend/services/project_admin_service.py b/backend/services/project_admin_service.py index 895fcb7dc5..ef8e93a92f 100644 --- a/backend/services/project_admin_service.py +++ b/backend/services/project_admin_service.py @@ -113,10 +113,10 @@ def _get_project_by_id(project_id: int) -> Project: return project @staticmethod - def get_project_dto_for_admin(project_id: int) -> ProjectDTO: + async def get_project_dto_for_admin(project_id: int, db: Database) -> ProjectDTO: """Get the project as DTO for project managers""" - project = ProjectAdminService._get_project_by_id(project_id) - return project.as_dto_for_admin(project_id) + project = await Project.exists(project_id, db) + return await Project.as_dto_for_admin(project_id, db) @staticmethod def update_project(project_dto: ProjectDTO, authenticated_user_id: int): diff --git a/backend/services/project_service.py b/backend/services/project_service.py index 6544a4e502..1c3db15ce7 100644 --- a/backend/services/project_service.py +++ b/backend/services/project_service.py @@ -41,6 +41,7 @@ from sqlalchemy import select from fastapi import HTTPException from databases import Database +import json summary_cache = TTLCache(maxsize=1024, ttl=600) @@ -92,10 +93,12 @@ def get_project_by_name(project_id: int) -> Project: return project + @staticmethod async def auto_unlock_tasks(project_id: int, session): await Task.auto_unlock_tasks(project_id, session) + @staticmethod def delete_tasks(project_id: int, tasks_ids): # Validate project exists. @@ -113,64 +116,48 @@ def delete_tasks(project_id: int, tasks_ids): # Delete task one by one. [t["obj"].delete() for t in tasks] - @staticmethod - def get_contribs_by_day(project_id: int) -> ProjectContribsDTO: - # Validate that project exists - project = ProjectService.get_project_by_id(project_id) - # Fetch all state change with date and task ID - stats = ( - session.query(TaskHistory).with_entities( - TaskHistory.action_text.label("action_text"), - func.DATE(TaskHistory.action_date).label("day"), - TaskHistory.task_id.label("task_id"), - ) - .filter(TaskHistory.project_id == project_id) - .filter( - TaskHistory.action == "STATE_CHANGE", - or_( - TaskHistory.action_text == "MAPPED", - TaskHistory.action_text == "VALIDATED", - TaskHistory.action_text == "INVALIDATED", - ), - ) - .group_by("action_text", "day", "task_id") - .order_by("day") - ).all() + @staticmethod + async def get_contribs_by_day(project_id: int, db: Database) -> ProjectContribsDTO: + project = await ProjectService.get_project_by_id(project_id, db) + query = """ + SELECT + action_text, + DATE(action_date) AS day, + task_id + FROM task_history + WHERE project_id = :project_id + AND action = 'STATE_CHANGE' + AND action_text IN ('MAPPED', 'VALIDATED', 'INVALIDATED') + GROUP BY action_text, day, task_id + ORDER BY day ASC + """ + rows = await db.fetch_all(query=query, values={"project_id": project_id}) contribs_dto = ProjectContribsDTO() - # Filter and store unique dates - dates = list(set(r[1] for r in stats)) - dates.sort( - reverse=False - ) # Why was this reversed? To have the dates in ascending order - dates_list = [] + dates = sorted({row["day"] for row in rows}) + cumulative_mapped = 0 cumulative_validated = 0 - # A hashmap to track task state change updates tasks = { "MAPPED": {"total": 0}, "VALIDATED": {"total": 0}, "INVALIDATED": {"total": 0}, } + dates_list = [] for date in dates: dto = ProjectContribDTO( - { - "date": date, - "mapped": 0, - "validated": 0, - "total_tasks": project.total_tasks, - } + date=date, + mapped=0, + validated=0, + total_tasks=project.total_tasks ) - # s -> ('LOCKED_FOR_MAPPING', datetime.date(2019, 4, 23), 1) - # s[0] -> action, s[1] -> date, s[2] -> task_id - values = [(s[0], s[2]) for s in stats if date == s[1]] - values.sort(reverse=True) # Most recent action comes first - for val in values: - task_id = val[1] - task_status = val[0] - + + values = [(row["action_text"], row["task_id"]) for row in rows if row["day"] == date] + values.sort(reverse=True) + + for task_status, task_id in values: if task_status == "MAPPED": if task_id not in tasks["MAPPED"]: tasks["MAPPED"][task_id] = 1 @@ -188,7 +175,7 @@ def get_contribs_by_day(project_id: int) -> ProjectContribsDTO: tasks["MAPPED"][task_id] = 1 tasks["MAPPED"]["total"] += 1 dto.mapped += 1 - else: + else: # "INVALIDATED" if task_id not in tasks["INVALIDATED"]: tasks["INVALIDATED"][task_id] = 1 tasks["INVALIDATED"]["total"] += 1 @@ -207,10 +194,10 @@ def get_contribs_by_day(project_id: int) -> ProjectContribsDTO: cumulative_validated = tasks["VALIDATED"]["total"] dto.cumulative_mapped = cumulative_mapped dto.cumulative_validated = cumulative_validated + dates_list.append(dto) contribs_dto.stats = dates_list - return contribs_dto @@ -289,16 +276,25 @@ def get_project_tasks( return project.tasks_as_geojson(task_ids_str, order_by, order_by_type, status) @staticmethod - def get_project_aoi(project_id): - project = ProjectService.get_project_by_id(project_id) - return project.get_aoi_geometry_as_geojson() + async def get_project_aoi(project_id, db: Database): + project = await Project.exists(project_id, db) + return await Project.get_aoi_geometry_as_geojson(project_id, db) + @staticmethod - def get_project_priority_areas(project_id): - project = ProjectService.get_project_by_id(project_id) - geojson_areas = [] - for priority_area in project.priority_areas: - geojson_areas.append(priority_area.get_as_geojson()) + async def get_project_priority_areas(project_id: int, db: Database) -> list: + project = await Project.exists(project_id, db) + + # Fetch the priority areas' geometries as GeoJSON + query = """ + SELECT ST_AsGeoJSON(pa.geometry) AS geojson + FROM priority_areas pa + JOIN project_priority_areas ppa ON pa.id = ppa.priority_area_id + WHERE ppa.project_id = :project_id; + """ + rows = await db.fetch_all(query, values={"project_id": project_id}) + geojson_areas = [json.loads(row['geojson']) for row in rows] if rows else [] + return geojson_areas @staticmethod diff --git a/backend/services/stats_service.py b/backend/services/stats_service.py index bf47deb43d..1866f6a139 100644 --- a/backend/services/stats_service.py +++ b/backend/services/stats_service.py @@ -1,3 +1,4 @@ +import datetime from typing import Optional from cachetools import TTLCache, cached from datetime import date, timedelta @@ -298,121 +299,98 @@ async def get_last_activity(project_id: int, db: Database) -> ProjectLastActivit @staticmethod - def get_user_contributions(project_id: int) -> ProjectContributionsDTO: - """Get all user contributions on a project""" - - mapped_stmt = ( - session.query(Task).with_entities( - Task.mapped_by, - func.count(Task.mapped_by).label("count"), - func.array_agg(Task.id).label("task_ids"), - ) - .filter(Task.project_id == project_id) - .filter(Task.task_status != TaskStatus.BADIMAGERY.value) - .group_by(Task.mapped_by) - .subquery() - ) - badimagery_stmt = ( - session.query(Task).with_entities( - Task.mapped_by, - func.count(Task.mapped_by).label("count"), - func.array_agg(Task.id).label("task_ids"), - ) - .filter(Task.project_id == project_id) - .filter(Task.task_status == TaskStatus.BADIMAGERY.value) - .group_by(Task.mapped_by) - .subquery() - ) - validated_stmt = ( - session.query(Task).with_entities( - Task.validated_by, - func.count(Task.validated_by).label("count"), - func.array_agg(Task.id).label("task_ids"), - ) - .filter(Task.project_id == project_id) - .group_by(Task.validated_by) - .subquery() - ) - - project_contributions = ( - session.query(TaskHistory).with_entities(TaskHistory.user_id) - .filter( - TaskHistory.project_id == project_id, TaskHistory.action != "COMMENT" - ) - .distinct(TaskHistory.user_id) - .subquery() - ) + async def get_user_contributions(project_id: int, db: Database) -> ProjectContributionsDTO: + # Query to get user contributions + query = """ + WITH mapped AS ( + SELECT + mapped_by AS user_id, + COUNT(mapped_by) AS count, + ARRAY_AGG(id) AS task_ids + FROM tasks + WHERE project_id = :project_id + AND task_status != :bad_imagery_status + GROUP BY mapped_by + ), + badimagery AS ( + SELECT + mapped_by AS user_id, + COUNT(mapped_by) AS count, + ARRAY_AGG(id) AS task_ids + FROM tasks + WHERE project_id = :project_id + AND task_status = :bad_imagery_status + GROUP BY mapped_by + ), + validated AS ( + SELECT + validated_by AS user_id, + COUNT(validated_by) AS count, + ARRAY_AGG(id) AS task_ids + FROM tasks + WHERE project_id = :project_id + GROUP BY validated_by + ), + project_contributions AS ( + SELECT DISTINCT user_id + FROM task_history + WHERE project_id = :project_id + AND action != 'COMMENT' + ) + SELECT + u.id, + u.username, + u.name, + u.mapping_level, + u.picture_url, + u.date_registered, + COALESCE(m.count, 0) AS mapped, + COALESCE(v.count, 0) AS validated, + COALESCE(b.count, 0) AS bad_imagery, + COALESCE(m.count, 0) + COALESCE(v.count, 0) + COALESCE(b.count, 0) AS total, + COALESCE(m.task_ids, '{}') AS mapped_tasks, + COALESCE(v.task_ids, '{}') AS validated_tasks, + COALESCE(b.task_ids, '{}') AS bad_imagery_tasks + FROM users u + JOIN project_contributions pc ON u.id = pc.user_id + LEFT JOIN mapped m ON u.id = m.user_id + LEFT JOIN badimagery b ON u.id = b.user_id + LEFT JOIN validated v ON u.id = v.user_id + ORDER BY total DESC; + """ - results = ( - session.query( - User.id, - User.username, - User.name, - User.mapping_level, - User.picture_url, - User.date_registered, - coalesce(mapped_stmt.c.count, 0).label("mapped"), - coalesce(validated_stmt.c.count, 0).label("validated"), - coalesce(badimagery_stmt.c.count, 0).label("bad_imagery"), - ( - coalesce(mapped_stmt.c.count, 0) - + coalesce(validated_stmt.c.count, 0) - + coalesce(badimagery_stmt.c.count, 0) - ).label("total"), - mapped_stmt.c.task_ids.label("mapped_tasks"), - validated_stmt.c.task_ids.label("validated_tasks"), - badimagery_stmt.c.task_ids.label("bad_imagery_tasks"), - ) - .join(project_contributions, User.id == project_contributions.c.user_id) - .outerjoin(mapped_stmt, User.id == mapped_stmt.c.mapped_by) - .outerjoin(badimagery_stmt, User.id == badimagery_stmt.c.mapped_by) - .outerjoin(validated_stmt, User.id == validated_stmt.c.validated_by) - .group_by( - User.id, - User.username, - User.name, - User.mapping_level, - User.picture_url, - User.date_registered, - mapped_stmt.c.count, - mapped_stmt.c.task_ids, - badimagery_stmt.c.count, - badimagery_stmt.c.task_ids, - validated_stmt.c.count, - validated_stmt.c.task_ids, - ) - .order_by(desc("total")) - .all() - ) + # Execute the query + rows = await db.fetch_all(query, values={ + "project_id": project_id, + "bad_imagery_status": TaskStatus.BADIMAGERY.value + }) + # Process the results into DTO contrib_dto = ProjectContributionsDTO() user_contributions = [ UserContribution( dict( - username=r.username, - name=r.name, - mapping_level=MappingLevel(r.mapping_level).name, - picture_url=r.picture_url, - mapped=r.mapped, - bad_imagery=r.bad_imagery, - validated=r.validated, - total=r.total, - mapped_tasks=r.mapped_tasks if r.mapped_tasks is not None else [], - bad_imagery_tasks=r.bad_imagery_tasks - if r.bad_imagery_tasks - else [], - validated_tasks=r.validated_tasks - if r.validated_tasks is not None - else [], - date_registered=r.date_registered.date(), + username=row['username'], + name=row['name'], + mapping_level=MappingLevel(row['mapping_level']).name, + picture_url=row['picture_url'], + mapped=row['mapped'], + bad_imagery=row['bad_imagery'], + validated=row['validated'], + total=row['total'], + mapped_tasks=row['mapped_tasks'] if row['mapped_tasks'] is not None else [], + bad_imagery_tasks=row['bad_imagery_tasks'] if row['bad_imagery_tasks'] else [], + validated_tasks=row['validated_tasks'] if row['validated_tasks'] is not None else [], + date_registered=row['date_registered'].date() if isinstance(row['date_registered'], datetime.datetime) else None, ) ) - for r in results + for row in rows ] contrib_dto.user_contributions = user_contributions return contrib_dto + @staticmethod @cached(homepage_stats_cache) def get_homepage_stats(abbrev=True, session=None) -> HomePageStatsDTO: