Fix invalid buck retrieve from new bucket upload(scrapyd) notificatio…

…n endpoint.
andrew-bierman · Jan 13, 2025 · b7abe96 · b7abe96
1 parent 77450f9
commit b7abe96
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 7 deletions.
diff --git a/packages/validations/src/validations/itemRoutesValidator.ts b/packages/validations/src/validations/itemRoutesValidator.ts
@@ -95,6 +95,13 @@ export const getSimilarItems = z.object({
   limit: z.number(),
 });
 
+export const importNotifiedETL = z.object({
+  bucket_name: z.string(),
+  file_key: z.string(),
+  spider_name: z.string(),
+});
+export type ImportNotifiedETL = z.infer<typeof importNotifiedETL>;
+
 export const importItemHeaders = z.object({
   Name: z.string(),
   Weight: z.string(),

diff --git a/server/src/controllers/item/importNotifiedETL.ts b/server/src/controllers/item/importNotifiedETL.ts
@@ -1,13 +1,17 @@
+import * as validator from '@packrat/validations';
 import {
   bulkAddItemsGlobalService,
   parseCSVData,
   fetchFromS3,
 } from '../../services/item/item.service';
 import { User } from '../../drizzle/methods/User';
+import { Context } from 'hono';
 
-export const importNotifiedETL = async (c) => {
-  const params = c.req.query();
-  const file_name = params.file_key;
+export const importNotifiedETL = async (c: Context) => {
+  const body = await c.req.json<validator.ImportNotifiedETL>();
+  const file_name = body.file_key;
+  const bucket_name = body.bucket_name;
+  const spider_name = body.spider_name;
 
   const endpoint = c.env.BUCKET_ENDPOINT;
   const bucket = c.env.BUCKET_NAME;
@@ -34,7 +38,7 @@ export const importNotifiedETL = async (c) => {
 
   try {
     const fileData = await fetchFromS3(
-      `${endpoint}/${bucket}/${file_name}`,
+      `${endpoint}/${bucket_name}/${file_name}`,
       method,
       service,
       region,

diff --git a/server/src/routes/itemRoutes.ts b/server/src/routes/itemRoutes.ts
@@ -46,10 +46,10 @@ router.get(
   tryCatchWrapper(importFromBucket),
 );
 
-router.get(
+// Convert to POST because it alters our db.
+router.post(
   '/importNotifiedETL',
-  // authTokenMiddleware,
-  // zodParser(validator.importNotifiedETL, 'query'),
+  zodParser(validator.importNotifiedETL, 'body'),
   tryCatchWrapper(importNotifiedETL),
 );