diff --git a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx index 51b69f2d6..fd531365e 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx @@ -7,20 +7,16 @@ import { DataAlertIcon, DataCheckIcon, ExcelFileIcon, - SearchOffIcon, WorldSearchIcon, } from "@/components/icons"; -import WizardSteps from "@/components/wizard-steps"; import { InventoryUserFileAttributes, - addFile, clear, removeFile, } from "@/features/city/inventoryDataSlice"; -import { setSubsector, clearSubsector } from "@/features/city/subsectorSlice"; +import { setSubsector } from "@/features/city/subsectorSlice"; import { useTranslation } from "@/i18n/client"; import { RootState } from "@/lib/store"; -import { ScopeAttributes } from "@/models/Scope"; import { api } from "@/services/api"; import { logger } from "@/services/logger"; import { bytesToMB, nameToI18NKey } from "@/util/helpers"; @@ -71,7 +67,6 @@ import { MdOutlineCheckCircle, MdOutlineEdit, MdOutlineHomeWork, - MdOutlineSkipNext, MdRefresh, } from "react-icons/md"; import { useDispatch, useSelector } from "react-redux"; @@ -85,7 +80,6 @@ import type { import AddFileDataModal from "@/components/Modals/add-file-data-modal"; import { InventoryValueAttributes } from "@/models/InventoryValue"; -import { UserFileAttributes } from "@/models/UserFile"; import { motion } from "framer-motion"; function getMailURI(locode?: string, sector?: string, year?: number): string { @@ -98,45 +92,41 @@ function getMailURI(locode?: string, sector?: string, year?: number): string { function SearchDataSourcesPrompt({ t, isSearching, - isDisabled, + isDisabled = false, onSearchClicked, }: { t: TFunction; isSearching: boolean; - isDisabled: boolean; + isDisabled?: boolean; onSearchClicked: () => void; }) { return ( - - - - - No External Data Sources Available - - + + + {t("wait-for-search")} ); @@ -413,11 +403,7 @@ export default function AddDataSteps({ } function onSearchDataSourcesClicked() { - if (inventoryProgress) { - loadDataSources({ inventoryId: inventoryProgress.inventory.inventoryId }); - } else { - console.error("Inventory progress is still loading!"); - } + loadDataSources({ inventoryId: inventory }); } const [selectedSubsector, setSelectedSubsector] = @@ -913,14 +899,13 @@ export default function AddDataSteps({ ) : dataSourcesError ? (
- ) : dataSources && dataSources.length === 0 ? ( + ) : dataSources && dataSources?.length === 0 ? ( { const inventory = await db.models.Inventory.findOne({ @@ -22,10 +21,6 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => { { model: DataSource, as: "dataSources", - where: { - startYear: { [Op.lte]: inventory.year }, - endYear: { [Op.gte]: inventory.year }, - }, include: [ { model: db.models.Scope, as: "scopes" }, { model: db.models.Publisher, as: "publisher" }, @@ -48,7 +43,7 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => { throw new createHttpError.NotFound("Sector not found"); } - const applicableSources = DataSourceService.filterSources( + const { applicableSources, removedSources } = DataSourceService.filterSources( inventory, sector.dataSources, ); @@ -69,5 +64,5 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => { ) ).filter((source) => !!source); - return NextResponse.json({ data: sourceData }); + return NextResponse.json({ data: sourceData, removedSources }); }); diff --git a/app/src/app/api/v0/datasource/[inventoryId]/route.ts b/app/src/app/api/v0/datasource/[inventoryId]/route.ts index ff60864e0..144177037 100644 --- a/app/src/app/api/v0/datasource/[inventoryId]/route.ts +++ b/app/src/app/api/v0/datasource/[inventoryId]/route.ts @@ -101,10 +101,6 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => { { model: DataSource, as: "dataSources", - where: { - startYear: { [Op.lte]: inventory.year }, - endYear: { [Op.gte]: inventory.year }, - }, include: [ { model: Scope, as: "scopes" }, { model: Publisher, as: "publisher" }, @@ -142,7 +138,10 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => { const sources = sectorSources .concat(subSectorSources) .concat(subCategorySources); - const applicableSources = DataSourceService.filterSources(inventory, sources); + const { applicableSources, removedSources } = DataSourceService.filterSources( + inventory, + sources, + ); // determine scaling factor for downscaled sources const { @@ -176,7 +175,7 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => { ) ).filter((source) => !!source); - return NextResponse.json({ data: sourceData }); + return NextResponse.json({ data: sourceData, removedSources }); }); const applySourcesRequest = z.object({ @@ -214,7 +213,10 @@ export const POST = apiHandler(async (req: NextRequest, { params }) => { if (!sources) { throw new createHttpError.NotFound("Sources not found"); } - const applicableSources = DataSourceService.filterSources(inventory, sources); + const { applicableSources, removedSources } = DataSourceService.filterSources( + inventory, + sources, + ); const applicableSourceIds = applicableSources.map( (source) => source.datasourceId, ); @@ -297,6 +299,12 @@ export const POST = apiHandler(async (req: NextRequest, { params }) => { }, {}); return NextResponse.json({ - data: { successful, failed, invalid: invalidSourceIds, issues }, + data: { + successful, + failed, + invalid: invalidSourceIds, + issues, + removedSources, + }, }); }); diff --git a/app/src/backend/DataSourceService.ts b/app/src/backend/DataSourceService.ts index 741a4be19..6129d5305 100644 --- a/app/src/backend/DataSourceService.ts +++ b/app/src/backend/DataSourceService.ts @@ -7,32 +7,73 @@ import createHttpError from "http-errors"; const EARTH_LOCATION = "EARTH"; +type RemovedSourceResult = { source: DataSource; reason: string }; +type FilterSourcesResult = { + applicableSources: DataSource[]; + removedSources: RemovedSourceResult[]; +}; + export default class DataSourceService { public static filterSources( inventory: Inventory, dataSources: DataSource[], - ): DataSource[] { + ): FilterSourcesResult { if (!inventory.city) { throw createHttpError.InternalServerError( "Inventory doesn't contain city data!", ); } + if (!inventory.year) { + throw createHttpError.InternalServerError( + "Inventory doesn't contain year!", + ); + } const { city } = inventory; - return dataSources.filter((source) => { + const removedSources: RemovedSourceResult[] = []; + const applicableSources = dataSources.filter((source) => { const locations = source.geographicalLocation?.split(","); if (locations?.includes(EARTH_LOCATION)) { return true; } + if (!source.startYear || !source.endYear) { + removedSources.push({ + source, + reason: "startYear or endYear missing in source", + }); + return false; + } + const isMatchingYearRange = + source.startYear <= inventory.year! && + source.endYear >= inventory.year!; + if (!isMatchingYearRange) { + removedSources.push({ + source, + reason: "inventory year not in [startYear, endYear] range of source", + }); + return false; + } + // TODO store locode for country and region as separate columns in City const countryLocode = city.locode?.split(" ")[0]; const isCountry = countryLocode && locations?.includes(countryLocode); const isRegion = city.region && locations?.includes(city.region); const isCity = city.locode && locations?.includes(city.locode); + const isMatchingLocation = isCountry || isRegion || isCity; + + if (!isMatchingLocation) { + removedSources.push({ + source, + reason: "geographicalLocation doesn't match inventory locodes", + }); + return false; + } - return isCountry || isRegion || isCity; + return true; }); + + return { applicableSources, removedSources }; } public static async retrieveGlobalAPISource( diff --git a/app/src/i18n/locales/en/data.json b/app/src/i18n/locales/en/data.json index ed6e9ace9..b62f73fe9 100644 --- a/app/src/i18n/locales/en/data.json +++ b/app/src/i18n/locales/en/data.json @@ -164,7 +164,7 @@ "delete-activity": "Delete activity", "delete-activities-prompt": "Are you sure you want to <2>permanently delete all activities in the commercial and institutional buildings sub-sector from the city's inventory data?", "delete-activity-prompt": "Are you sure you want to <2> permanently delete this activity from the city's inventory data?", - + "skip-step-button": "Skip this step", "save-continue-button": "Save and Continue", "updated-every": "Updated every", @@ -228,6 +228,7 @@ "incomplete": "Incomplete", "search-available-datasets": "Search for available datasets", "searching": "Searching...", + "wait-for-search": "Please wait while we search for external data sources that match your inventory needs.", "no-external-sources": "It seemes like there are no external sources at this moment.", "report-any-sources": "if you know any, <2>please report this and we'll prioritize your request.", "unsaved-changes": "Unsaved Changes",