Merge pull request #570 from Open-Earth-Foundation/fix/source-filtering

Fix broken data sources section and add debugging tools for filtered out sources

app/src/app/[lng]/[inventory]/data/[step]/page.tsx

@@ -7,20 +7,16 @@ import {
   DataAlertIcon,
   DataCheckIcon,
   ExcelFileIcon,
-  SearchOffIcon,
   WorldSearchIcon,
 } from "@/components/icons";
-import WizardSteps from "@/components/wizard-steps";
 import {
   InventoryUserFileAttributes,
-  addFile,
   clear,
   removeFile,
 } from "@/features/city/inventoryDataSlice";
-import { setSubsector, clearSubsector } from "@/features/city/subsectorSlice";
+import { setSubsector } from "@/features/city/subsectorSlice";
 import { useTranslation } from "@/i18n/client";
 import { RootState } from "@/lib/store";
-import { ScopeAttributes } from "@/models/Scope";
 import { api } from "@/services/api";
 import { logger } from "@/services/logger";
 import { bytesToMB, nameToI18NKey } from "@/util/helpers";
@@ -71,7 +67,6 @@ import {
   MdOutlineCheckCircle,
   MdOutlineEdit,
   MdOutlineHomeWork,
-  MdOutlineSkipNext,
   MdRefresh,
 } from "react-icons/md";
 import { useDispatch, useSelector } from "react-redux";
@@ -85,7 +80,6 @@ import type {
 
 import AddFileDataModal from "@/components/Modals/add-file-data-modal";
 import { InventoryValueAttributes } from "@/models/InventoryValue";
-import { UserFileAttributes } from "@/models/UserFile";
 import { motion } from "framer-motion";
 
 function getMailURI(locode?: string, sector?: string, year?: number): string {
@@ -98,45 +92,41 @@ function getMailURI(locode?: string, sector?: string, year?: number): string {
 function SearchDataSourcesPrompt({
   t,
   isSearching,
-  isDisabled,
+  isDisabled = false,
   onSearchClicked,
 }: {
   t: TFunction;
   isSearching: boolean;
-  isDisabled: boolean;
+  isDisabled?: boolean;
   onSearchClicked: () => void;
 }) {
   return (
     <Flex align="center" direction="column">
-      <Box p="16px" mb="24px" borderRadius="full" bg="background.neutral">
-        <SearchOffIcon />
-      </Box>
-      <Text
-        fontFamily="heading"
-        fontSize="title.lg"
-        fontWeight="600"
-        color="interactive.control"
-      >
-        No External Data Sources Available
-      </Text>
-      <Text
-        color="content.tertiary"
-        align="center"
-        size="sm"
-        variant="spaced"
-        w="550px"
-        mt="8px"
+      <Icon
+        as={WorldSearchIcon}
+        boxSize={20}
+        color="interactive.secondary"
+        borderRadius="full"
+        p={4}
+        bgColor="background.neutral"
+        mb={6}
+      />
+      <Button
+        variant="solid"
+        leftIcon={<SearchIcon boxSize={6} />}
+        isLoading={isSearching}
+        isDisabled={isDisabled}
+        loadingText={t("searching")}
+        onClick={onSearchClicked}
+        mb={2}
+        px={6}
+        h={16}
+        py={4}
       >
-        <Text>{t("no-external-sources")}</Text>
-        <Text fontWeight="semibold">
-          <Trans t={t} i18nKey="report-any-sources">
-            if you know any,{" "}
-            <Link href="/" color="content.link" textDecor="underline">
-              please report this
-            </Link>{" "}
-            and we&apos;ll prioritize your request.
-          </Trans>
-        </Text>
+        {t("search-available-datasets")}
+      </Button>
+      <Text color="content.tertiary" align="center" size="sm" variant="spaced">
+        {t("wait-for-search")}
       </Text>
     </Flex>
   );
@@ -413,11 +403,7 @@ export default function AddDataSteps({
   }
 
   function onSearchDataSourcesClicked() {
-    if (inventoryProgress) {
-      loadDataSources({ inventoryId: inventoryProgress.inventory.inventoryId });
-    } else {
-      console.error("Inventory progress is still loading!");
-    }
+    loadDataSources({ inventoryId: inventory });
   }
 
   const [selectedSubsector, setSelectedSubsector] =
@@ -913,14 +899,13 @@ export default function AddDataSteps({
             <SearchDataSourcesPrompt
               t={t}
               isSearching={areDataSourcesLoading}
-              isDisabled={!inventoryProgress}
               onSearchClicked={onSearchDataSourcesClicked}
             />
           ) : dataSourcesError ? (
             <Center>
               <WarningIcon boxSize={8} color="semantic.danger" />
             </Center>
-          ) : dataSources && dataSources.length === 0 ? (
+          ) : dataSources && dataSources?.length === 0 ? (
             <NoDataSourcesMessage
               t={t}
               sector={currentStep.referenceNumber}

app/src/app/api/v0/datasource/[inventoryId]/[sectorId]/route.ts

@@ -5,7 +5,6 @@ import { DataSource } from "@/models/DataSource";
 import { apiHandler } from "@/util/api";
 import createHttpError from "http-errors";
 import { NextRequest, NextResponse } from "next/server";
-import { Op } from "sequelize";
 
 export const GET = apiHandler(async (_req: NextRequest, { params }) => {
   const inventory = await db.models.Inventory.findOne({
@@ -22,10 +21,6 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
       {
         model: DataSource,
         as: "dataSources",
-        where: {
-          startYear: { [Op.lte]: inventory.year },
-          endYear: { [Op.gte]: inventory.year },
-        },
         include: [
           { model: db.models.Scope, as: "scopes" },
           { model: db.models.Publisher, as: "publisher" },
@@ -48,7 +43,7 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
     throw new createHttpError.NotFound("Sector not found");
   }
 
-  const applicableSources = DataSourceService.filterSources(
+  const { applicableSources, removedSources } = DataSourceService.filterSources(
     inventory,
     sector.dataSources,
   );
@@ -69,5 +64,5 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
     )
   ).filter((source) => !!source);
 
-  return NextResponse.json({ data: sourceData });
+  return NextResponse.json({ data: sourceData, removedSources });
 });

app/src/app/api/v0/datasource/[inventoryId]/route.ts

@@ -101,10 +101,6 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
     {
       model: DataSource,
       as: "dataSources",
-      where: {
-        startYear: { [Op.lte]: inventory.year },
-        endYear: { [Op.gte]: inventory.year },
-      },
       include: [
         { model: Scope, as: "scopes" },
         { model: Publisher, as: "publisher" },
@@ -142,7 +138,10 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
   const sources = sectorSources
     .concat(subSectorSources)
     .concat(subCategorySources);
-  const applicableSources = DataSourceService.filterSources(inventory, sources);
+  const { applicableSources, removedSources } = DataSourceService.filterSources(
+    inventory,
+    sources,
+  );
 
   // determine scaling factor for downscaled sources
   const {
@@ -176,7 +175,7 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
     )
   ).filter((source) => !!source);
 
-  return NextResponse.json({ data: sourceData });
+  return NextResponse.json({ data: sourceData, removedSources });
 });
 
 const applySourcesRequest = z.object({
@@ -214,7 +213,10 @@ export const POST = apiHandler(async (req: NextRequest, { params }) => {
   if (!sources) {
     throw new createHttpError.NotFound("Sources not found");
   }
-  const applicableSources = DataSourceService.filterSources(inventory, sources);
+  const { applicableSources, removedSources } = DataSourceService.filterSources(
+    inventory,
+    sources,
+  );
   const applicableSourceIds = applicableSources.map(
     (source) => source.datasourceId,
   );
@@ -297,6 +299,12 @@ export const POST = apiHandler(async (req: NextRequest, { params }) => {
     }, {});
 
   return NextResponse.json({
-    data: { successful, failed, invalid: invalidSourceIds, issues },
+    data: {
+      successful,
+      failed,
+      invalid: invalidSourceIds,
+      issues,
+      removedSources,
+    },
   });
 });

app/src/backend/DataSourceService.ts

@@ -7,32 +7,73 @@ import createHttpError from "http-errors";
 
 const EARTH_LOCATION = "EARTH";
 
+type RemovedSourceResult = { source: DataSource; reason: string };
+type FilterSourcesResult = {
+  applicableSources: DataSource[];
+  removedSources: RemovedSourceResult[];
+};
+
 export default class DataSourceService {
   public static filterSources(
     inventory: Inventory,
     dataSources: DataSource[],
-  ): DataSource[] {
+  ): FilterSourcesResult {
     if (!inventory.city) {
       throw createHttpError.InternalServerError(
         "Inventory doesn't contain city data!",
       );
     }
+    if (!inventory.year) {
+      throw createHttpError.InternalServerError(
+        "Inventory doesn't contain year!",
+      );
+    }
     const { city } = inventory;
 
-    return dataSources.filter((source) => {
+    const removedSources: RemovedSourceResult[] = [];
+    const applicableSources = dataSources.filter((source) => {
       const locations = source.geographicalLocation?.split(",");
       if (locations?.includes(EARTH_LOCATION)) {
         return true;
       }
 
+      if (!source.startYear || !source.endYear) {
+        removedSources.push({
+          source,
+          reason: "startYear or endYear missing in source",
+        });
+        return false;
+      }
+      const isMatchingYearRange =
+        source.startYear <= inventory.year! &&
+        source.endYear >= inventory.year!;
+      if (!isMatchingYearRange) {
+        removedSources.push({
+          source,
+          reason: "inventory year not in [startYear, endYear] range of source",
+        });
+        return false;
+      }
+
       // TODO store locode for country and region as separate columns in City
       const countryLocode = city.locode?.split(" ")[0];
       const isCountry = countryLocode && locations?.includes(countryLocode);
       const isRegion = city.region && locations?.includes(city.region);
       const isCity = city.locode && locations?.includes(city.locode);
+      const isMatchingLocation = isCountry || isRegion || isCity;
+
+      if (!isMatchingLocation) {
+        removedSources.push({
+          source,
+          reason: "geographicalLocation doesn't match inventory locodes",
+        });
+        return false;
+      }
 
-      return isCountry || isRegion || isCity;
+      return true;
     });
+
+    return { applicableSources, removedSources };
   }
 
   public static async retrieveGlobalAPISource(

app/src/i18n/locales/en/data.json

@@ -164,7 +164,7 @@
   "delete-activity": "Delete activity",
   "delete-activities-prompt": "Are you sure you want to <2>permanently delete </2> all activities in the commercial and institutional buildings sub-sector from the city's inventory data?",
   "delete-activity-prompt": "Are you sure you want to <2> permanently delete </2> this activity from the city's inventory data?",
-  
+
   "skip-step-button": "Skip this step",
   "save-continue-button": "Save and Continue",
   "updated-every": "Updated every",
@@ -228,6 +228,7 @@
   "incomplete": "Incomplete",
   "search-available-datasets": "Search for available datasets",
   "searching": "Searching...",
+  "wait-for-search": "Please wait while we search for external data sources that match your inventory needs.",
   "no-external-sources": "It seemes like there are no external sources at this moment.",
   "report-any-sources": "if you know any, <2>please report this</2> and we'll prioritize your request.",
   "unsaved-changes": "Unsaved Changes",