Merge pull request #130 from Vizzuality/cms/improve-csv-import-error-…

…management Add csv column validation to csv import endpoints
Vizzuality · Sep 12, 2024 · 45c09f6 · 45c09f6
2 parents bce5028 + b629bc0
commit 45c09f6
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 1 deletion.
diff --git a/cms/src/api/collaborator/services/collaborator.ts b/cms/src/api/collaborator/services/collaborator.ts
@@ -11,11 +11,30 @@ export default factories.createCoreService('api::collaborator.collaborator', {
   async parseAndPrepareCSV(file: any, author: number | null = null): Promise<{ csvData: string; rowCount: number }> {
     // Read and parse the CSV file
     const fileContent = fs.readFileSync(file.path, 'utf8');
+
+    const allowedColumns = ['name', 'link', 'type'];
+
     const records: any[] = csv.parse(fileContent, {
       columns: true,
       skip_empty_lines: true,
     });
 
+    if (records.length === 0) {
+      throw new Error('CSV file is empty');
+    }
+
+    const csvColumns = Object.keys(records[0]);
+
+    const invalidColumns = csvColumns.filter(col => !allowedColumns.includes(col));
+    if (invalidColumns.length > 0) {
+      throw new Error(`Invalid columns detected: ${invalidColumns.join(', ')}`);
+    }
+
+    const missingColumns = allowedColumns.filter(col => !csvColumns.includes(col));
+    if (missingColumns.length > 0) {
+      throw new Error(`Missing required columns: ${missingColumns.join(', ')}`);
+    }
+
     // Process each row
     const updatedRecords = records.map((row: any) => {
       const publishedAt = new Date().toISOString();

diff --git a/cms/src/api/other-tool/services/other-tool.ts b/cms/src/api/other-tool/services/other-tool.ts
@@ -25,12 +25,32 @@ export default factories.createCoreService('api::other-tool.other-tool', {
   async parseAndReplaceIds(file: any, author: number | null = null): Promise<{ csvData: string; rowCount: number }> {
     // Read and parse the CSV file
     const fileContent = fs.readFileSync(file.path, 'utf8');
+
+    const allowedColumns = ['name', 'description', 'link', 'other_tools_category'];
+
     const records: any[] = csv.parse(fileContent, {
       columns: true,
       skip_empty_lines: true,
     });
 
-    // Process each row
+    if (records.length === 0) {
+      throw new Error('CSV file is empty');
+    }
+
+    const csvColumns = Object.keys(records[0]);
+
+    // Check for any columns not allowed
+    const invalidColumns = csvColumns.filter(col => !allowedColumns.includes(col));
+    if (invalidColumns.length > 0) {
+      throw new Error(`Invalid columns detected: ${invalidColumns.join(', ')}`);
+    }
+
+    // Check for missing columns and throw an error if any are missing
+    const missingColumns = allowedColumns.filter(col => !csvColumns.includes(col));
+    if (missingColumns.length > 0) {
+      throw new Error(`Missing required columns: ${missingColumns.join(', ')}`);
+    }
+
     const updatedRecords = await Promise.all(records.map(async (row: any) => {
       const categoryNames = row.other_tools_category.split(';').map(name => name.trim());
 

diff --git a/cms/src/api/project/services/project.ts b/cms/src/api/project/services/project.ts
@@ -57,11 +57,30 @@ export default factories.createCoreService('api::project.project', {
   async parseAndReplaceIds(file, author = null) {
     // Read and parse the CSV file
     const fileContent = fs.readFileSync(file.path, 'utf8');
+
+    const allowedColumns = ['name', 'highlight', 'status', 'objective', 'amount', 'countries', 'source_country', 'sdgs', 'pillar', 'organization_type', 'info', 'funding'];
+
     const records: ProjectRow[] = csv.parse(fileContent, {
       columns: true,
       skip_empty_lines: true,
     });
 
+    if (records.length === 0) {
+      throw new Error('CSV file is empty');
+    }
+
+    const csvColumns = Object.keys(records[0]);
+
+    const invalidColumns = csvColumns.filter(col => !allowedColumns.includes(col));
+    if (invalidColumns.length > 0) {
+      throw new Error(`Invalid columns detected: ${invalidColumns.join(', ')}`);
+    }
+
+    const missingColumns = allowedColumns.filter(col => !csvColumns.includes(col));
+    if (missingColumns.length > 0) {
+      throw new Error(`Missing required columns: ${missingColumns.join(', ')}`);
+    }
+
     // Process each row
     const updatedRecords = await Promise.all(records.map(async (row: ProjectRow) => {
       const countryNames = row.countries.split(';').map(name => name.trim());