feat(api): return list of removed sources and reasons for easier debu…

…gging
Open-Earth-Foundation · Jul 17, 2024 · 4ee13df · 4ee13df
1 parent be61abb
commit 4ee13df
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 18 deletions.
diff --git a/app/src/app/api/v0/datasource/[inventoryId]/[sectorId]/route.ts b/app/src/app/api/v0/datasource/[inventoryId]/[sectorId]/route.ts
@@ -5,7 +5,6 @@ import { DataSource } from "@/models/DataSource";
 import { apiHandler } from "@/util/api";
 import createHttpError from "http-errors";
 import { NextRequest, NextResponse } from "next/server";
-import { Op } from "sequelize";
 
 export const GET = apiHandler(async (_req: NextRequest, { params }) => {
   const inventory = await db.models.Inventory.findOne({
@@ -22,10 +21,6 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
       {
         model: DataSource,
         as: "dataSources",
-        where: {
-          startYear: { [Op.lte]: inventory.year },
-          endYear: { [Op.gte]: inventory.year },
-        },
         include: [
           { model: db.models.Scope, as: "scopes" },
           { model: db.models.Publisher, as: "publisher" },
@@ -48,7 +43,7 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
     throw new createHttpError.NotFound("Sector not found");
   }
 
-  const applicableSources = DataSourceService.filterSources(
+  const { applicableSources, removedSources } = DataSourceService.filterSources(
     inventory,
     sector.dataSources,
   );
@@ -69,5 +64,5 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
     )
   ).filter((source) => !!source);
 
-  return NextResponse.json({ data: sourceData });
+  return NextResponse.json({ data: sourceData, removedSources });
 });
diff --git a/app/src/app/api/v0/datasource/[inventoryId]/route.ts b/app/src/app/api/v0/datasource/[inventoryId]/route.ts
@@ -101,10 +101,6 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
     {
       model: DataSource,
       as: "dataSources",
-      where: {
-        startYear: { [Op.lte]: inventory.year },
-        endYear: { [Op.gte]: inventory.year },
-      },
       include: [
         { model: Scope, as: "scopes" },
         { model: Publisher, as: "publisher" },
@@ -142,7 +138,10 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
   const sources = sectorSources
     .concat(subSectorSources)
     .concat(subCategorySources);
-  const applicableSources = DataSourceService.filterSources(inventory, sources);
+  const { applicableSources, removedSources } = DataSourceService.filterSources(
+    inventory,
+    sources,
+  );
 
   // determine scaling factor for downscaled sources
   const {
@@ -176,7 +175,7 @@ export const GET = apiHandler(async (_req: NextRequest, { params }) => {
     )
   ).filter((source) => !!source);
 
-  return NextResponse.json({ data: sourceData });
+  return NextResponse.json({ data: sourceData, removedSources });
 });
 
 const applySourcesRequest = z.object({
@@ -214,7 +213,10 @@ export const POST = apiHandler(async (req: NextRequest, { params }) => {
   if (!sources) {
     throw new createHttpError.NotFound("Sources not found");
   }
-  const applicableSources = DataSourceService.filterSources(inventory, sources);
+  const { applicableSources, removedSources } = DataSourceService.filterSources(
+    inventory,
+    sources,
+  );
   const applicableSourceIds = applicableSources.map(
     (source) => source.datasourceId,
   );
@@ -297,6 +299,12 @@ export const POST = apiHandler(async (req: NextRequest, { params }) => {
     }, {});
 
   return NextResponse.json({
-    data: { successful, failed, invalid: invalidSourceIds, issues },
+    data: {
+      successful,
+      failed,
+      invalid: invalidSourceIds,
+      issues,
+      removedSources,
+    },
   });
 });
diff --git a/app/src/backend/DataSourceService.ts b/app/src/backend/DataSourceService.ts
@@ -7,32 +7,73 @@ import createHttpError from "http-errors";
 
 const EARTH_LOCATION = "EARTH";
 
+type RemovedSourceResult = { source: DataSource; reason: string };
+type FilterSourcesResult = {
+  applicableSources: DataSource[];
+  removedSources: RemovedSourceResult[];
+};
+
 export default class DataSourceService {
   public static filterSources(
     inventory: Inventory,
     dataSources: DataSource[],
-  ): DataSource[] {
+  ): FilterSourcesResult {
     if (!inventory.city) {
       throw createHttpError.InternalServerError(
         "Inventory doesn't contain city data!",
       );
     }
+    if (!inventory.year) {
+      throw createHttpError.InternalServerError(
+        "Inventory doesn't contain year!",
+      );
+    }
     const { city } = inventory;
 
-    return dataSources.filter((source) => {
+    const removedSources: RemovedSourceResult[] = [];
+    const applicableSources = dataSources.filter((source) => {
       const locations = source.geographicalLocation?.split(",");
       if (locations?.includes(EARTH_LOCATION)) {
         return true;
       }
 
+      if (!source.startYear || !source.endYear) {
+        removedSources.push({
+          source,
+          reason: "startYear or endYear missing in source",
+        });
+        return false;
+      }
+      const isMatchingYearRange =
+        source.startYear <= inventory.year! &&
+        source.endYear >= inventory.year!;
+      if (!isMatchingYearRange) {
+        removedSources.push({
+          source,
+          reason: "inventory year not in [startYear, endYear] range of source",
+        });
+        return false;
+      }
+
       // TODO store locode for country and region as separate columns in City
       const countryLocode = city.locode?.split(" ")[0];
       const isCountry = countryLocode && locations?.includes(countryLocode);
       const isRegion = city.region && locations?.includes(city.region);
       const isCity = city.locode && locations?.includes(city.locode);
+      const isMatchingLocation = isCountry || isRegion || isCity;
+
+      if (!isMatchingLocation) {
+        removedSources.push({
+          source,
+          reason: "geographicalLocation doesn't match inventory locodes",
+        });
+        return false;
+      }
 
-      return isCountry || isRegion || isCity;
+      return true;
     });
+
+    return { applicableSources, removedSources };
   }
 
   public static async retrieveGlobalAPISource(