diff --git a/app/migrations/20250227103030-data-source-priority.cjs b/app/migrations/20250227103030-data-source-priority.cjs new file mode 100644 index 000000000..8db8f6e25 --- /dev/null +++ b/app/migrations/20250227103030-data-source-priority.cjs @@ -0,0 +1,15 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn("DataSourceI18n", "priority", { + type: Sequelize.DOUBLE, + nullable: true, + }); + }, + + async down(queryInterface) { + await queryInterface.removeColumn("DataSourceI18n", "priority"); + }, +}; diff --git a/app/scripts/catalogue-sync.ts b/app/scripts/catalogue-sync.ts index e3a8529ac..6b7bdd5e0 100644 --- a/app/scripts/catalogue-sync.ts +++ b/app/scripts/catalogue-sync.ts @@ -36,6 +36,7 @@ interface Source { subsector_id?: string; created?: Date; last_updated?: Date; + priority?: number; } function snakeToCamel(str: string): string { @@ -181,7 +182,6 @@ async function syncDataCatalogue() { return result as DataSourceCreationAttributes; }); - console.dir(sources); logger.debug("Saving sources..."); /* diff --git a/app/src/app/api/v0/admin/bulk/route.ts b/app/src/app/api/v0/admin/bulk/route.ts new file mode 100644 index 000000000..720b29029 --- /dev/null +++ b/app/src/app/api/v0/admin/bulk/route.ts @@ -0,0 +1,18 @@ +import AdminService from "@/backend/AdminService"; +import { apiHandler } from "@/util/api"; +import { NextResponse } from "next/server"; +import { z } from "zod"; + +const createBulkInventoriesRequest = z.object({ + cityLocodes: z.array(z.string()), // List of city locodes + emails: z.array(z.string().email()), // Comma separated list of emails to invite to the all of the created inventories + years: z.array(z.number().int().positive()), // List of years to create inventories for (can be comma separated input, multiple select dropdown etc., so multiple years can be chosen) + scope: z.enum(["gpc_basic", "gpc_basic_plus"]), // Scope selection (gpc_basic or gpc_basic_plus) + gwp: z.enum(["AR5", "AR6"]), // GWP selection (AR5 or AR6) +}); + +export const POST = apiHandler(async (req, { session }) => { + const props = createBulkInventoriesRequest.parse(await req.json()); + const result = await AdminService.createBulkInventories(props, session); + return NextResponse.json(result); +}); diff --git a/app/src/app/api/v0/city/[city]/user/[user]/route.ts b/app/src/app/api/v0/city/[city]/user/[user]/route.ts index 67005c9a2..5d8ee6d6f 100644 --- a/app/src/app/api/v0/city/[city]/user/[user]/route.ts +++ b/app/src/app/api/v0/city/[city]/user/[user]/route.ts @@ -19,8 +19,8 @@ const updateUserRequest = z.object({ role: z.nativeEnum(Roles), }); -export const PATCH = apiHandler(async (_req, { params, session }) => { - const body = updateUserRequest.parse(await _req.json()); +export const PATCH = apiHandler(async (req, { params, session }) => { + const body = updateUserRequest.parse(await req.json()); let user = await db.models.User.findOne({ where: { userId: params.user } }); if (!user) { diff --git a/app/src/app/api/v0/datasource/[inventoryId]/route.ts b/app/src/app/api/v0/datasource/[inventoryId]/route.ts index 34256f4f9..fc3eb6290 100644 --- a/app/src/app/api/v0/datasource/[inventoryId]/route.ts +++ b/app/src/app/api/v0/datasource/[inventoryId]/route.ts @@ -1,4 +1,7 @@ -import DataSourceService from "@/backend/DataSourceService"; +import DataSourceService, { + downscaledByCountryPopulation, + downscaledByRegionPopulation, +} from "@/backend/DataSourceService"; import { db } from "@/models"; import { City } from "@/models/City"; import { DataSourceI18n as DataSource } from "@/models/DataSourceI18n"; @@ -9,84 +12,7 @@ import { SubSector } from "@/models/SubSector"; import { apiHandler } from "@/util/api"; import createHttpError from "http-errors"; import { NextRequest, NextResponse } from "next/server"; -import { Op } from "sequelize"; import { z } from "zod"; -import { logger } from "@/services/logger"; -import { Publisher } from "@/models/Publisher"; -import { findClosestYear, PopulationEntry } from "@/util/helpers"; -import { PopulationAttributes } from "@/models/Population"; -import { Inventory } from "@/models/Inventory"; -import { maxPopulationYearDifference } from "@/util/constants"; - -const downscaledByCountryPopulation = "global_api_downscaled_by_population"; -const downscaledByRegionPopulation = - "global_api_downscaled_by_region_population"; -const populationScalingRetrievalMethods = [ - downscaledByCountryPopulation, - downscaledByRegionPopulation, -]; - -async function findPopulationScaleFactors( - inventory: Inventory, - sources: DataSource[], -) { - let countryPopulationScaleFactor = 1; - let regionPopulationScaleFactor = 1; - let populationIssue: string | null = null; - if ( - sources.some((source) => - populationScalingRetrievalMethods.includes(source.retrievalMethod ?? ""), - ) - ) { - const populations = await db.models.Population.findAll({ - where: { - cityId: inventory.cityId, - year: { - [Op.between]: [ - inventory.year! - maxPopulationYearDifference, - inventory.year! + maxPopulationYearDifference, - ], - }, - }, - order: [["year", "DESC"]], // favor more recent population entries - }); - const cityPopulations = populations.filter((pop) => !!pop.population); - const cityPopulation = findClosestYear( - cityPopulations as PopulationEntry[], - inventory.year!, - ); - const countryPopulations = populations.filter( - (pop) => !!pop.countryPopulation, - ); - const countryPopulation = findClosestYear( - countryPopulations as PopulationEntry[], - inventory.year!, - ) as PopulationAttributes; - const regionPopulations = populations.filter( - (pop) => !!pop.regionPopulation, - ); - const regionPopulation = findClosestYear( - regionPopulations as PopulationEntry[], - inventory.year!, - ) as PopulationAttributes; - // TODO allow country downscaling to work if there is no region population? - if (!cityPopulation || !countryPopulation || !regionPopulation) { - // City is missing population/ region population/ country population for a year close to the inventory year - populationIssue = "missing-population"; // translation key - } else { - countryPopulationScaleFactor = - cityPopulation.population / countryPopulation.countryPopulation!; - regionPopulationScaleFactor = - cityPopulation.population / regionPopulation.regionPopulation!; - } - } - - return { - countryPopulationScaleFactor, - regionPopulationScaleFactor, - populationIssue, - }; -} export const GET = apiHandler(async (_req: NextRequest, { params }) => { const inventory = await db.models.Inventory.findOne({ @@ -97,48 +23,7 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => { throw new createHttpError.NotFound("Inventory not found"); } - const include = [ - { - model: DataSource, - as: "dataSources", - include: [ - { model: Scope, as: "scopes" }, - { model: Publisher, as: "publisher" }, - { - model: InventoryValue, - as: "inventoryValues", - required: false, - where: { inventoryId: params.inventoryId }, - }, - { model: SubSector, as: "subSector" }, - { - model: SubCategory, - as: "subCategory", - include: [ - { model: SubSector, as: "subsector" }, - { model: Scope, as: "scope" }, - ], - }, - ], - }, - ]; - - const sectors = await db.models.Sector.findAll({ include }); - const subSectors = await db.models.SubSector.findAll({ include }); - const subCategories = await db.models.SubCategory.findAll({ include }); - - const sectorSources = sectors.flatMap((sector) => sector.dataSources); - const subSectorSources = subSectors.flatMap( - (subSector) => subSector.dataSources, - ); - const subCategorySources = subCategories.flatMap( - (subCategory) => subCategory.dataSources, - ); - - const sources = sectorSources - .concat(subSectorSources) - .concat(subCategorySources); - + const sources = await DataSourceService.findAllSources(params.inventoryId); const { applicableSources, removedSources } = DataSourceService.filterSources( inventory, sources, @@ -149,7 +34,10 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => { countryPopulationScaleFactor, regionPopulationScaleFactor, populationIssue, - } = await findPopulationScaleFactors(inventory, applicableSources); + } = await DataSourceService.findPopulationScaleFactors( + inventory, + applicableSources, + ); // TODO add query parameter to make this optional? const sourceData = await Promise.all( @@ -234,59 +122,20 @@ export const POST = apiHandler(async (req: NextRequest, { params }) => { // TODO check if the user has made manual edits that would be overwritten // TODO create new versioning record - const { - countryPopulationScaleFactor, - regionPopulationScaleFactor, - populationIssue, - } = await findPopulationScaleFactors(inventory, applicableSources); + const populationScaleFactors = + await DataSourceService.findPopulationScaleFactors( + inventory, + applicableSources, + ); // download source data and apply in database const sourceResults = await Promise.all( applicableSources.map(async (source) => { - const result: { id: string; success: boolean; issue?: string } = { - id: source.datasourceId, - success: true, - issue: undefined, - }; - - if (source.retrievalMethod === "global_api") { - const sourceStatus = await DataSourceService.applyGlobalAPISource( - source, - inventory, - ); - if (typeof sourceStatus === "string") { - result.issue = sourceStatus; - result.success = false; - } - } else if ( - populationScalingRetrievalMethods.includes(source.retrievalMethod ?? "") - ) { - if (populationIssue) { - result.issue = populationIssue; - result.success = false; - return result; - } - let scaleFactor = 1.0; - if (source.retrievalMethod === downscaledByCountryPopulation) { - scaleFactor = countryPopulationScaleFactor; - } else if (source.retrievalMethod === downscaledByRegionPopulation) { - scaleFactor = regionPopulationScaleFactor; - } - const sourceStatus = await DataSourceService.applyGlobalAPISource( - source, - inventory, - scaleFactor, - ); - if (typeof sourceStatus === "string") { - result.issue = sourceStatus; - result.success = false; - } - } else { - result.issue = `Unsupported retrieval method ${source.retrievalMethod} for data source ${source.datasourceId}`; - logger.error(result.issue); - result.success = false; - } - + const result = await DataSourceService.applySource( + source, + inventory, + populationScaleFactors, + ); return result; }), ); diff --git a/app/src/backend/AdminService.ts b/app/src/backend/AdminService.ts new file mode 100644 index 000000000..ff1edf3f4 --- /dev/null +++ b/app/src/backend/AdminService.ts @@ -0,0 +1,248 @@ +import { AppSession } from "@/lib/auth"; +import { db } from "@/models"; +import { logger } from "@/services/logger"; +import { Roles } from "@/util/types"; +import createHttpError from "http-errors"; +import { randomUUID } from "node:crypto"; +import DataSourceService from "./DataSourceService"; +import { City } from "@/models/City"; +import { groupBy } from "@/util/helpers"; +import OpenClimateService from "./OpenClimateService"; +import { Op } from "sequelize"; + +export interface BulkInventoryProps { + cityLocodes: string[]; // List of city locodes + emails: string[]; // Comma separated list of emails to invite to the all of the created inventories + years: number[]; // List of years to create inventories for + scope: "gpc_basic" | "gpc_basic_plus"; // Scope selection (gpc_basic or gpc_basic_plus) + gwp: "AR5" | "AR6"; // GWP selection (AR5 or AR6) +} + +export interface CreateBulkInventoriesResponse { + errors: { locode: string; error: any }[]; + results: { locode: string; result: string[] }[]; +} + +const DEFAULT_PRIORITY = 0; // 10 is the highest priority + +export default class AdminService { + public static async createBulkInventories( + props: BulkInventoryProps, + session: AppSession | null, + ): Promise { + // Ensure user has admin role + const isAdmin = session?.user?.role === Roles.Admin; + if (!isAdmin) { + throw new createHttpError.Unauthorized("Not signed in as an admin"); + } + + const errors: { locode: string; error: any }[] = []; + const results: { locode: string; result: string[] }[] = []; + + // Find user accounts to add to created inventories + const users = await db.models.User.findAll({ + where: { email: { [Op.in]: props.emails } }, + }); + if (users.length !== props.emails.length) { + throw new createHttpError.BadRequest( + "Not all users to be added to inventories were found", + ); + } + + // Bulk create inventories + logger.info( + `Creating bulk inventories for cities ${props.cityLocodes} and years ${props.years}`, + ); + for (const cityLocode of props.cityLocodes) { + const cityName = "Test"; // TODO query from OpenClimate + const city = await db.models.City.create({ + cityId: randomUUID(), + locode: cityLocode, + name: cityName, + }); + + // add users to the city + await db.models.CityUser.bulkCreate( + users.map((user) => ({ + cityUserId: randomUUID(), + cityId: city.cityId, + userId: user.userId, + })), + ); + + logger.info("Creating inventories for city " + cityLocode); + const inventories = props.years.map((year) => ({ + inventoryId: randomUUID(), + cityLocode, + year, + scope: props.scope, + gwp: props.gwp, + cityId: city.cityId, + })); + try { + const createdInventories = + await db.models.Inventory.bulkCreate(inventories); + results.push({ + locode: cityLocode, + result: createdInventories.map((inventory) => inventory.inventoryId), + }); + } catch (err) { + errors.push({ locode: cityLocode, error: err }); + } + + for (const inventory of inventories) { + const populationErrors = await this.createPopulationEntries( + cityLocode, + inventory.year, + city.cityId, + ); + errors.push(...populationErrors); + + // Connect all data sources, rank them by priority, check if they connect + const sourceErrors = await this.connectAllDataSources( + inventory.inventoryId, + cityLocode, + ); + errors.push(...sourceErrors); + } + } + + return { errors, results }; + } + + private static async createPopulationEntries( + cityLocode: string, + inventoryYear: number, + cityId: string, + ) { + const errors: { locode: string; error: string }[] = []; + + // query population data from OpenClimate and save in Population table + const populationData = await OpenClimateService.getPopulationData( + cityLocode, + inventoryYear, + ); + if (populationData.error) { + errors.push({ locode: cityLocode, error: populationData.error }); + } + if ( + !populationData.cityPopulation || + !populationData.cityPopulationYear || + !populationData.countryPopulation || + !populationData.countryPopulationYear || + !populationData.regionPopulation || + !populationData.regionPopulationYear + ) { + errors.push({ + locode: cityLocode, + error: `Population data incomplete for city ${cityLocode} and inventory year ${inventoryYear}`, + }); + return errors; + } + + // they might be for the same year, but that is not guaranteed (because of data availability) + await db.models.Population.upsert({ + population: populationData.cityPopulation, + cityId, + year: populationData.cityPopulationYear, + }); + await db.models.Population.upsert({ + countryPopulation: populationData.countryPopulation, + cityId, + year: populationData.countryPopulationYear, + }); + await db.models.Population.upsert({ + regionPopulation: populationData.regionPopulation, + cityId, + year: populationData.regionPopulationYear, + }); + + return errors; + } + + private static async connectAllDataSources( + inventoryId: string, + cityLocode: string, + ): Promise<{ locode: string; error: string }[]> { + const errors: any[] = []; + const inventory = await db.models.Inventory.findOne({ + where: { inventoryId }, + include: [{ model: City, as: "city" }], + }); + if (!inventory) { + throw new createHttpError.NotFound("Inventory not found"); + } + // Find all data sources for the inventory + const sources = await DataSourceService.findAllSources(inventoryId); + // Filter by locally available criteria (e.g. geographical location, year of inventory etc.) + const { applicableSources } = DataSourceService.filterSources( + inventory, + sources, + ); + + // group sources by subsector so we can prioritize for each choice individually + const sourcesBySubsector = groupBy( + applicableSources, + (source) => source.subsectorId ?? source.subcategoryId ?? "unknown", + ); + delete sourcesBySubsector["unknown"]; + + const populationScaleFactors = + await DataSourceService.findPopulationScaleFactors( + inventory, + applicableSources, + ); + + await Promise.all( + Object.entries(sourcesBySubsector).map(async ([subSector, sources]) => { + // Sort each group by priority field + const prioritizedSources = sources.sort( + (a, b) => + (b.priority ?? DEFAULT_PRIORITY) - (a.priority ?? DEFAULT_PRIORITY), + ); + + // Try one after another until one connects successfully + let isSuccessful = false; + for (const source of prioritizedSources) { + const data = await DataSourceService.retrieveGlobalAPISource( + source, + inventory, + ); + if (data instanceof String || typeof data === "string") { + errors.push({ + locode: cityLocode, + error: `Failed to fetch source - ${source.datasourceId}: ${data}`, + }); + } else { + // save data source to DB + // download source data and apply in database + const result = await DataSourceService.applySource( + source, + inventory, + populationScaleFactors, + ); + if (result.success) { + isSuccessful = true; + break; + } else { + logger.error( + `Failed to apply source ${source.datasourceId}: ${result.issue}`, + ); + } + } + + if (!isSuccessful) { + const message = `Wasn't able to find a data source for subsector ${subSector}`; + logger.error(`${cityLocode} - ${message}`); + errors.push({ + locode: cityLocode, + error: message, + }); + } + } + }), + ); + + return errors; + } +} diff --git a/app/src/backend/DataSourceService.ts b/app/src/backend/DataSourceService.ts index 5be76ba15..4dd5b63f7 100644 --- a/app/src/backend/DataSourceService.ts +++ b/app/src/backend/DataSourceService.ts @@ -1,23 +1,221 @@ import { db } from "@/models"; -import type { DataSourceI18n as DataSource } from "@/models/DataSourceI18n"; +import { DataSourceI18n as DataSource } from "@/models/DataSourceI18n"; import { Inventory } from "@/models/Inventory"; import { randomUUID } from "crypto"; import createHttpError from "http-errors"; import Decimal from "decimal.js"; import { decimalToBigInt } from "@/util/big_int"; -import type { SubSector } from "@/models/SubSector"; +import { SubSector } from "@/models/SubSector"; import { DataSourceActivityDataRecord } from "@/app/[lng]/[inventory]/data/[step]/types"; +import { InventoryValue } from "@/models/InventoryValue"; +import { Publisher } from "@/models/Publisher"; +import { Scope } from "@/models/Scope"; +import { SubCategory } from "@/models/SubCategory"; +import { logger } from "@/services/logger"; +import { findClosestYear, PopulationEntry } from "@/util/helpers"; +import { PopulationAttributes } from "@/models/Population"; +import { maxPopulationYearDifference } from "@/util/constants"; +import { Op } from "sequelize"; const EARTH_LOCATION = "EARTH"; export type RemovedSourceResult = { source: DataSource; reason: string }; export type FailedSourceResult = { source: DataSource; error: string }; -type FilterSourcesResult = { +export type FilterSourcesResult = { applicableSources: DataSource[]; removedSources: RemovedSourceResult[]; }; +export type ApplySourceResult = { + id: string; + success: boolean; + issue?: string; +}; +export type PopulationScaleFactorResponse = { + countryPopulationScaleFactor: number; + regionPopulationScaleFactor: number; + populationIssue: string | null; +}; + +export const downscaledByCountryPopulation = + "global_api_downscaled_by_population"; +export const downscaledByRegionPopulation = + "global_api_downscaled_by_region_population"; +export const populationScalingRetrievalMethods = [ + downscaledByCountryPopulation, + downscaledByRegionPopulation, +]; export default class DataSourceService { + public static async findAllSources( + inventoryId: string, + ): Promise { + const include = [ + { + model: DataSource, + as: "dataSources", + include: [ + { model: Scope, as: "scopes" }, + { model: Publisher, as: "publisher" }, + { + model: InventoryValue, + as: "inventoryValues", + required: false, + where: { inventoryId }, + }, + { model: SubSector, as: "subSector" }, + { + model: SubCategory, + as: "subCategory", + include: [ + { model: SubSector, as: "subsector" }, + { model: Scope, as: "scope" }, + ], + }, + ], + }, + ]; + + const sectors = await db.models.Sector.findAll({ include }); + const subSectors = await db.models.SubSector.findAll({ include }); + const subCategories = await db.models.SubCategory.findAll({ include }); + + const sectorSources = sectors.flatMap((sector) => sector.dataSources); + const subSectorSources = subSectors.flatMap( + (subSector) => subSector.dataSources, + ); + const subCategorySources = subCategories.flatMap( + (subCategory) => subCategory.dataSources, + ); + + const sources = sectorSources + .concat(subSectorSources) + .concat(subCategorySources); + + return sources; + } + + public static async findPopulationScaleFactors( + inventory: Inventory, + sources: DataSource[], + ) { + let countryPopulationScaleFactor = 1; + let regionPopulationScaleFactor = 1; + let populationIssue: string | null = null; + if ( + sources.some((source) => + populationScalingRetrievalMethods.includes( + source.retrievalMethod ?? "", + ), + ) + ) { + const populations = await db.models.Population.findAll({ + where: { + cityId: inventory.cityId, + year: { + [Op.between]: [ + inventory.year! - maxPopulationYearDifference, + inventory.year! + maxPopulationYearDifference, + ], + }, + }, + order: [["year", "DESC"]], // favor more recent population entries + }); + const cityPopulations = populations.filter((pop) => !!pop.population); + const cityPopulation = findClosestYear( + cityPopulations as PopulationEntry[], + inventory.year!, + ); + const countryPopulations = populations.filter( + (pop) => !!pop.countryPopulation, + ); + const countryPopulation = findClosestYear( + countryPopulations as PopulationEntry[], + inventory.year!, + ) as PopulationAttributes; + const regionPopulations = populations.filter( + (pop) => !!pop.regionPopulation, + ); + const regionPopulation = findClosestYear( + regionPopulations as PopulationEntry[], + inventory.year!, + ) as PopulationAttributes; + // TODO allow country downscaling to work if there is no region population? + if (!cityPopulation || !countryPopulation || !regionPopulation) { + // City is missing population/ region population/ country population for a year close to the inventory year + populationIssue = "missing-population"; // translation key + } else { + countryPopulationScaleFactor = + cityPopulation.population / countryPopulation.countryPopulation!; + regionPopulationScaleFactor = + cityPopulation.population / regionPopulation.regionPopulation!; + } + } + + return { + countryPopulationScaleFactor, + regionPopulationScaleFactor, + populationIssue, + }; + } + + public static async applySource( + source: DataSource, + inventory: Inventory, + populationScaleFactors: PopulationScaleFactorResponse, // obtained from findPopulationScaleFactors + ): Promise { + const result: ApplySourceResult = { + id: source.datasourceId, + success: true, + issue: undefined, + }; + + const { + countryPopulationScaleFactor, + regionPopulationScaleFactor, + populationIssue, + } = populationScaleFactors; + + if (source.retrievalMethod === "global_api") { + const sourceStatus = await DataSourceService.applyGlobalAPISource( + source, + inventory, + ); + if (typeof sourceStatus === "string") { + result.issue = sourceStatus; + result.success = false; + } + } else if ( + populationScalingRetrievalMethods.includes(source.retrievalMethod ?? "") + ) { + if (populationIssue) { + result.issue = populationIssue; + result.success = false; + return result; + } + let scaleFactor = 1.0; + if (source.retrievalMethod === downscaledByCountryPopulation) { + scaleFactor = countryPopulationScaleFactor; + } else if (source.retrievalMethod === downscaledByRegionPopulation) { + scaleFactor = regionPopulationScaleFactor; + } + const sourceStatus = await DataSourceService.applyGlobalAPISource( + source, + inventory, + scaleFactor, + ); + if (typeof sourceStatus === "string") { + result.issue = sourceStatus; + result.success = false; + } + } else { + result.issue = `Unsupported retrieval method ${source.retrievalMethod} for data source ${source.datasourceId}`; + logger.error(result.issue); + result.success = false; + } + + return result; + } + public static filterSources( inventory: Inventory, dataSources: DataSource[], @@ -113,9 +311,13 @@ export default class DataSourceService { } if (typeof data.totals !== "object") { - const message = "Incorrect response from Global API for URL: " + url; - console.error(message, data); - return message; + if (data.detail === "No data available") { + return "Source doesn't have data available for this input"; + } else { + const message = "Incorrect response from Global API for URL: " + url; + console.error(message, data); + return message; + } } return data; diff --git a/app/src/backend/OpenClimateService.ts b/app/src/backend/OpenClimateService.ts new file mode 100644 index 000000000..ce1a15f39 --- /dev/null +++ b/app/src/backend/OpenClimateService.ts @@ -0,0 +1,109 @@ +import { logger } from "@/services/logger"; +import { findClosestYear, PopulationEntry } from "@/util/helpers"; + +const OPENCLIMATE_BASE_URL = + process.env.NEXT_PUBLIC_OPENCLIMATE_API_URL || + "https://app.openclimate.network"; + +const numberOfYearsDisplayed = 10; + +interface PopulationDataResult { + error?: string; + cityPopulation?: number; + cityPopulationYear?: number; + regionPopulation?: number; + regionPopulationYear?: number; + countryPopulation?: number; + countryPopulationYear?: number; +} + +type FetchPopulationResult = PopulationEntry & { data: any }; + +export default class OpenClimateService { + public static async getPopulationData( + inventoryLocode: string, + inventoryYear: number, + ): Promise { + const url = OPENCLIMATE_BASE_URL + "/api/v1/actor/"; + const result: PopulationDataResult = {}; + + try { + const cityResult = await this.fetchPopulation( + inventoryLocode, + inventoryYear, + url, + ); + if (!cityResult) { + result.error = "No city population result found!"; + return result; + } + result.cityPopulation = cityResult.population; + result.cityPopulationYear = cityResult.year; + + const countryLocode = + inventoryLocode && inventoryLocode.length > 0 + ? inventoryLocode.split(" ")[0] + : null; + if (!countryLocode) { + result.error = `Invalid locode supplied, doesn\'t have a country locode: ${inventoryLocode}`; + return result; + } + + const countryResult = await this.fetchPopulation( + countryLocode, + inventoryYear, + url, + ); + if (!countryResult) { + result.error = "No country population result found!"; + return result; + } + result.countryPopulation = countryResult.population; + result.countryPopulationYear = countryResult.year; + + const regionLocode = cityResult.data.data.is_part_of; + if (!regionLocode) { + result.error = `City ${inventoryLocode} does not have a region locode in OpenClimate`; + return result; + } + + const regionResult = await this.fetchPopulation( + regionLocode, + inventoryYear, + url, + ); + if (!regionResult) { + result.error = "No region population result found!"; + return result; + } + result.regionPopulation = regionResult.population; + result.regionPopulationYear = regionResult.year; + } catch (err) { + const message = `Failed to query population data for city ${inventoryLocode} and year ${inventoryYear} from URL ${url}: ${err}`; + logger.error(message); + result.error = message; + } + + return result; + } + + private static async fetchPopulation( + actorLocode: string, + inventoryYear: number, + baseUrl: string, + ): Promise { + const request = await fetch(baseUrl + actorLocode); + const data = await request.json(); + + const result = findClosestYear( + data.data.population, + inventoryYear, + numberOfYearsDisplayed, + ); + if (!result) { + return null; + } + + return { ...result, data }; + } +} diff --git a/app/src/models/DataSourceI18n.ts b/app/src/models/DataSourceI18n.ts index 2bdc57447..938e5b3ad 100644 --- a/app/src/models/DataSourceI18n.ts +++ b/app/src/models/DataSourceI18n.ts @@ -63,6 +63,7 @@ export interface DataSourceI18nAttributes { subcategoryId?: string; created?: Date; lastUpdated?: Date; + priority?: number; // 1-10, 10 being the highest priority } export type DataSourcePk = "datasourceId"; @@ -94,6 +95,7 @@ export type DataSourceOptionalAttributes = | "sectorId" | "subsectorId" | "subcategoryId" + | "priority" | "created" | "lastUpdated"; export type DataSourceI18nCreationAttributes = Optional< @@ -132,6 +134,7 @@ export class DataSourceI18n sectorId?: string; subsectorId?: string; subcategoryId?: string; + priority?: number; created?: Date; lastUpdated?: Date; @@ -908,6 +911,10 @@ export class DataSourceI18n }, field: "subcategory_id", }, + priority: { + type: DataTypes.DOUBLE, + allowNull: true, + }, }, { sequelize, diff --git a/app/tests/api/admin.jest.ts b/app/tests/api/admin.jest.ts index 098a24919..1f3c2bc29 100644 --- a/app/tests/api/admin.jest.ts +++ b/app/tests/api/admin.jest.ts @@ -1,4 +1,5 @@ import { POST as changeRole } from "@/app/api/v0/auth/role/route"; +import { POST as createBulkInventories } from "@/app/api/v0/admin/bulk/route"; import { db } from "@/models"; import { beforeAll, @@ -12,6 +13,9 @@ import { import { mockRequest, setupTests, testUserData, testUserID } from "../helpers"; import { AppSession, Auth } from "@/lib/auth"; import { Roles } from "@/util/types"; +import { BulkInventoryProps } from "@/backend/AdminService"; +import { Op } from "sequelize"; +import _ from "lodash"; const mockSession: AppSession = { user: { id: testUserID, role: Roles.User }, @@ -21,6 +25,13 @@ const mockAdminSession: AppSession = { user: { id: testUserID, role: Roles.Admin }, expires: "1h", }; +const mockBulkInventoriesRequest: BulkInventoryProps = { + cityLocodes: ["US NYC", "DE BER", "BR AAX"], + emails: [testUserData.email], + years: [2022, 2023, 2024], + scope: "gpc_basic_plus", + gwp: "AR6", +}; describe("Admin API", () => { let prevGetServerSession = Auth.getServerSession; @@ -45,6 +56,69 @@ describe("Admin API", () => { }); }); + it("should allow creating bulk inventories for admin users", async () => { + const req = mockRequest(mockBulkInventoriesRequest); + Auth.getServerSession = jest.fn(() => Promise.resolve(mockAdminSession)); + const res = await createBulkInventories(req, { params: {} }); + expect(res.status).toBe(200); + const body = await res.json(); + console.dir(body.errors.slice(0, 10)); + // expect(body.errors.length).toBe(0); // TODO ignore missing data from Global API in errors + expect(body.results.length).toBe( + mockBulkInventoriesRequest.cityLocodes.length, + ); + + // check inventories were created + const inventoryIds = body.results.flatMap((result: any) => result.result); + expect(inventoryIds.length).toBe( + mockBulkInventoriesRequest.years.length * + mockBulkInventoriesRequest.cityLocodes.length, + ); + const cities = await db.models.City.findAll({ + attributes: ["cityId", "locode"], + include: { + model: db.models.Inventory, + as: "inventories", + where: { inventoryId: { [Op.in]: inventoryIds } }, + }, + }); + const cityIds = cities.map((city) => city.cityId); + const uniqueCityIds = [...new Set(cityIds)]; + + // check population entries for inventory + for (const cityId of uniqueCityIds) { + const populationEntries = await db.models.Population.findAll({ + where: { cityId }, + }); + const hasCityPopulation = _.some( + populationEntries, + (entry) => entry.population != null, + ); + const hasCountryPopulation = _.some( + populationEntries, + (entry) => entry.countryPopulation != null, + ); + const hasRegionPopulation = _.some( + populationEntries, + (entry) => entry.regionPopulation != null, + ); + expect(hasCityPopulation).toBe(true); + expect(hasCountryPopulation).toBe(true); + expect(hasRegionPopulation).toBe(true); + + // check that users were added to inventory (without sending the emails) + const cityUsers = await db.models.CityUser.findAll({ + where: { cityId }, + }); + expect(cityUsers.length).toBe(1); + for (const cityUser of cityUsers) { + expect(cityUser.userId).toBe(testUserID); + } + + // TODO check all data sources for inventory are connected + } + }, 60000); + it("should change the user role when logged in as admin", async () => { const req = mockRequest({ email: testUserData.email, @@ -93,7 +167,7 @@ describe("Admin API", () => { const res = await changeRole(req, { params: {} }); expect(res.status).toBe(400); - const req2 = mockRequest({ email: "not-an-email", role: "Admin" }); + const req2 = mockRequest({ email: "not-an-email", role: "admin" }); const res2 = await changeRole(req2, { params: {} }); expect(res2.status).toBe(400); diff --git a/app/tests/api/datasource.jest.ts b/app/tests/api/datasource.jest.ts index 56167f463..b12d0aae4 100644 --- a/app/tests/api/datasource.jest.ts +++ b/app/tests/api/datasource.jest.ts @@ -69,6 +69,17 @@ const mockGlobalApiResponses = [ }, ]; +async function cleanupDatabase() { + await db.models.Inventory.destroy({ where: { year: inventoryData.year } }); + await cascadeDeleteDataSource({ + [Op.or]: [literal(`dataset_name ->> 'en' LIKE 'XX_INVENTORY_TEST_%'`)], + }); + await db.models.City.destroy({ where: { locode } }); + await db.models.SubCategory.destroy({ where: { subcategoryName } }); + await db.models.SubSector.destroy({ where: { subsectorName } }); + await db.models.Sector.destroy({ where: { sectorName } }); +} + describe("DataSource API", () => { let city: City; let inventory: Inventory; @@ -80,12 +91,8 @@ describe("DataSource API", () => { Auth.getServerSession = jest.fn(() => Promise.resolve(mockSession)); await db.initialize(); + await cleanupDatabase(); - await db.models.Inventory.destroy({ where: { year: inventoryData.year } }); - await cascadeDeleteDataSource({ - [Op.or]: [literal(`dataset_name ->> 'en' LIKE 'XX_INVENTORY_TEST_%'`)], - }); - await db.models.City.destroy({ where: { locode } }); city = await db.models.City.create({ cityId: randomUUID(), locode, @@ -96,9 +103,6 @@ describe("DataSource API", () => { userId: testUserID, cityId: city.cityId, }); - await db.models.SubCategory.destroy({ where: { subcategoryName } }); - await db.models.SubSector.destroy({ where: { subsectorName } }); - await db.models.Sector.destroy({ where: { sectorName } }); inventory = await db.models.Inventory.create({ ...inventoryData, @@ -147,6 +151,7 @@ describe("DataSource API", () => { }); afterAll(async () => { + await cleanupDatabase(); Auth.getServerSession = prevGetServerSession; if (db.sequelize) await db.sequelize.close(); });