diff --git a/apps/vite/src/routeTree.gen.ts b/apps/vite/src/routeTree.gen.ts index 8e9dfcd1e..968fb50c5 100644 --- a/apps/vite/src/routeTree.gen.ts +++ b/apps/vite/src/routeTree.gen.ts @@ -174,163 +174,94 @@ const ProfileSettingsIndexLazyRoute = ProfileSettingsIndexLazyImport.update({ declare module '@tanstack/react-router' { interface FileRoutesByPath { '/': { - id: '/' - path: '/' - fullPath: '/' preLoaderRoute: typeof IndexImport parentRoute: typeof rootRoute } '/destination/query': { - id: '/destination/query' - path: '/destination/query' - fullPath: '/destination/query' preLoaderRoute: typeof DestinationQueryLazyImport parentRoute: typeof rootRoute } '/item/$itemId': { - id: '/item/$itemId' - path: '/item/$itemId' - fullPath: '/item/$itemId' preLoaderRoute: typeof ItemItemIdLazyImport parentRoute: typeof rootRoute } '/pack/$id': { - id: '/pack/$id' - path: '/pack/$id' - fullPath: '/pack/$id' preLoaderRoute: typeof PackIdLazyImport parentRoute: typeof rootRoute } '/pack/create': { - id: '/pack/create' - path: '/pack/create' - fullPath: '/pack/create' preLoaderRoute: typeof PackCreateLazyImport parentRoute: typeof rootRoute } '/profile/$id': { - id: '/profile/$id' - path: '/profile/$id' - fullPath: '/profile/$id' preLoaderRoute: typeof ProfileIdLazyImport parentRoute: typeof rootRoute } '/trip/$tripId': { - id: '/trip/$tripId' - path: '/trip/$tripId' - fullPath: '/trip/$tripId' preLoaderRoute: typeof TripTripIdLazyImport parentRoute: typeof rootRoute } '/trip/create': { - id: '/trip/create' - path: '/trip/create' - fullPath: '/trip/create' preLoaderRoute: typeof TripCreateLazyImport parentRoute: typeof rootRoute } '/about/': { - id: '/about/' - path: '/about' - fullPath: '/about' preLoaderRoute: typeof AboutIndexLazyImport parentRoute: typeof rootRoute } '/appearance/': { - id: '/appearance/' - path: '/appearance' - fullPath: '/appearance' preLoaderRoute: typeof AppearanceIndexLazyImport parentRoute: typeof rootRoute } '/dashboard/': { - id: '/dashboard/' - path: '/dashboard' - fullPath: '/dashboard' preLoaderRoute: typeof DashboardIndexLazyImport parentRoute: typeof rootRoute } '/feed/': { - id: '/feed/' - path: '/feed' - fullPath: '/feed' preLoaderRoute: typeof FeedIndexLazyImport parentRoute: typeof rootRoute } '/items/': { - id: '/items/' - path: '/items' - fullPath: '/items' preLoaderRoute: typeof ItemsIndexLazyImport parentRoute: typeof rootRoute } '/map/': { - id: '/map/' - path: '/map' - fullPath: '/map' preLoaderRoute: typeof MapIndexLazyImport parentRoute: typeof rootRoute } '/maps/': { - id: '/maps/' - path: '/maps' - fullPath: '/maps' preLoaderRoute: typeof MapsIndexLazyImport parentRoute: typeof rootRoute } '/packs/': { - id: '/packs/' - path: '/packs' - fullPath: '/packs' preLoaderRoute: typeof PacksIndexLazyImport parentRoute: typeof rootRoute } '/password-reset/': { - id: '/password-reset/' - path: '/password-reset' - fullPath: '/password-reset' preLoaderRoute: typeof PasswordResetIndexLazyImport parentRoute: typeof rootRoute } '/privacy/': { - id: '/privacy/' - path: '/privacy' - fullPath: '/privacy' preLoaderRoute: typeof PrivacyIndexLazyImport parentRoute: typeof rootRoute } '/profile/': { - id: '/profile/' - path: '/profile' - fullPath: '/profile' preLoaderRoute: typeof ProfileIndexLazyImport parentRoute: typeof rootRoute } '/register/': { - id: '/register/' - path: '/register' - fullPath: '/register' preLoaderRoute: typeof RegisterIndexLazyImport parentRoute: typeof rootRoute } '/sign-in/': { - id: '/sign-in/' - path: '/sign-in' - fullPath: '/sign-in' preLoaderRoute: typeof SignInIndexLazyImport parentRoute: typeof rootRoute } '/trips/': { - id: '/trips/' - path: '/trips' - fullPath: '/trips' preLoaderRoute: typeof TripsIndexLazyImport parentRoute: typeof rootRoute } '/profile/settings/': { - id: '/profile/settings/' - path: '/profile/settings' - fullPath: '/profile/settings' preLoaderRoute: typeof ProfileSettingsIndexLazyImport parentRoute: typeof rootRoute } @@ -339,7 +270,7 @@ declare module '@tanstack/react-router' { // Create and export the route tree -export const routeTree = rootRoute.addChildren({ +export const routeTree = rootRoute.addChildren([ IndexRoute, DestinationQueryLazyRoute, ItemItemIdLazyRoute, @@ -363,110 +294,6 @@ export const routeTree = rootRoute.addChildren({ SignInIndexLazyRoute, TripsIndexLazyRoute, ProfileSettingsIndexLazyRoute, -}) +]) /* prettier-ignore-end */ - -/* ROUTE_MANIFEST_START -{ - "routes": { - "__root__": { - "filePath": "__root.tsx", - "children": [ - "/", - "/destination/query", - "/item/$itemId", - "/pack/$id", - "/pack/create", - "/profile/$id", - "/trip/$tripId", - "/trip/create", - "/about/", - "/appearance/", - "/dashboard/", - "/feed/", - "/items/", - "/map/", - "/maps/", - "/packs/", - "/password-reset/", - "/privacy/", - "/profile/", - "/register/", - "/sign-in/", - "/trips/", - "/profile/settings/" - ] - }, - "/": { - "filePath": "index.tsx" - }, - "/destination/query": { - "filePath": "destination/query.lazy.tsx" - }, - "/item/$itemId": { - "filePath": "item/$itemId.lazy.tsx" - }, - "/pack/$id": { - "filePath": "pack/$id.lazy.tsx" - }, - "/pack/create": { - "filePath": "pack/create.lazy.tsx" - }, - "/profile/$id": { - "filePath": "profile/$id.lazy.tsx" - }, - "/trip/$tripId": { - "filePath": "trip/$tripId.lazy.tsx" - }, - "/trip/create": { - "filePath": "trip/create.lazy.tsx" - }, - "/about/": { - "filePath": "about/index.lazy.tsx" - }, - "/appearance/": { - "filePath": "appearance/index.lazy.tsx" - }, - "/dashboard/": { - "filePath": "dashboard/index.lazy.tsx" - }, - "/feed/": { - "filePath": "feed/index.lazy.tsx" - }, - "/items/": { - "filePath": "items/index.lazy.tsx" - }, - "/map/": { - "filePath": "map/index.lazy.tsx" - }, - "/maps/": { - "filePath": "maps/index.lazy.tsx" - }, - "/packs/": { - "filePath": "packs/index.lazy.tsx" - }, - "/password-reset/": { - "filePath": "password-reset/index.lazy.tsx" - }, - "/privacy/": { - "filePath": "privacy/index.lazy.tsx" - }, - "/profile/": { - "filePath": "profile/index.lazy.tsx" - }, - "/register/": { - "filePath": "register/index.lazy.tsx" - }, - "/sign-in/": { - "filePath": "sign-in/index.lazy.tsx" - }, - "/trips/": { - "filePath": "trips/index.lazy.tsx" - }, - "/profile/settings/": { - "filePath": "profile/settings/index.lazy.tsx" - } - } -} -ROUTE_MANIFEST_END */ diff --git a/packages/app/modules/item/components/ImportItemGlobal.tsx b/packages/app/modules/item/components/ImportItemGlobal.tsx index e08292db6..935e3eabf 100644 --- a/packages/app/modules/item/components/ImportItemGlobal.tsx +++ b/packages/app/modules/item/components/ImportItemGlobal.tsx @@ -8,7 +8,7 @@ export const ImportItemGlobal = () => { const ownerId = authUser?.id; if (!authUser) { - return null; // or some fallback + return null; } const { setIsModalOpen } = useModal(); diff --git a/packages/app/modules/item/hooks/useImportFromBucket.ts b/packages/app/modules/item/hooks/useImportFromBucket.ts new file mode 100644 index 000000000..72297b582 --- /dev/null +++ b/packages/app/modules/item/hooks/useImportFromBucket.ts @@ -0,0 +1,65 @@ +import { useCallback } from 'react'; +import { queryTrpc } from 'app/trpc'; +import { useOfflineQueue } from 'app/hooks/offline'; +import { useItemsUpdater } from './useItemsUpdater'; + +interface State { + items?: Array<{ id: string }>; +} + +export const useImportFromBucket = () => { + const utils = queryTrpc.useContext(); + const { mutateAsync } = queryTrpc.importFromBucket.useMutation(); + const { isConnected, addOfflineRequest } = useOfflineQueue(); + const updateItems = useItemsUpdater(); + + const handleImportFromBucket = useCallback( + async ({ directory, ownerId }, onSuccess) => { + if (isConnected) { + try { + const data = await mutateAsync({ directory, ownerId }); + const newItems = Array.isArray(data) ? data : []; + updateItems((prevState: State = {}) => { + const prevItems = Array.isArray(prevState.items) + ? prevState.items + : []; + return { + ...prevState, + items: [...newItems, ...prevItems], + }; + }); + + utils.getItemsGlobally.invalidate(); + utils.getItemsGlobally.refetch(); + onSuccess(); + } catch (error) { + console.error('Error fetching items:', error); + } + } else { + addOfflineRequest('importFromBucket', { directory, ownerId }); + + updateItems((prevState: State = {}) => { + onSuccess(); + const prevItems = Array.isArray(prevState.items) + ? prevState.items + : []; + + return { + ...prevState, + items: [ + { + directory, + ownerId, + global: true, + }, + ...prevItems, + ], + }; + }); + } + }, + [updateItems, isConnected, mutateAsync, utils, addOfflineRequest], + ); + + return { handleImportFromBucket }; +}; diff --git a/packages/app/modules/item/hooks/useImportItem.ts b/packages/app/modules/item/hooks/useImportItem.ts index 91fbcbbc1..f673aa7d8 100644 --- a/packages/app/modules/item/hooks/useImportItem.ts +++ b/packages/app/modules/item/hooks/useImportItem.ts @@ -14,27 +14,33 @@ export const useImportItem = () => { const updateItems = useItemsUpdater(); const handleImportNewItems = useCallback( - (newItem) => { + (newItem, onSuccess) => { if (isConnected) { return mutate(newItem, { onSuccess: () => { + updateItems((prevState: State = {}) => { + const prevItems = Array.isArray(prevState.items) + ? prevState.items + : []; + return { + ...prevState, + items: [newItem, ...prevItems], // Use the data returned from the server + }; + }); utils.getItemsGlobally.invalidate(); + onSuccess(); + }, + onError: (error) => { + console.error('Error adding item:', error); }, }); } addOfflineRequest('addItemGlobal', newItem); - updateItems((prevState: State = {}) => { - const prevItems = Array.isArray(prevState.items) ? prevState.items : []; - - return { - ...prevState, - items: [newItem, ...prevItems], - }; - }); + // Optionally, handle offline case here if needed }, - [updateItems], + [updateItems, isConnected, mutate, utils, addOfflineRequest], ); return { handleImportNewItems }; diff --git a/packages/app/modules/pack/hooks/useImportPackItem.ts b/packages/app/modules/pack/hooks/useImportPackItem.ts index 400d19964..b7afa3f14 100644 --- a/packages/app/modules/pack/hooks/useImportPackItem.ts +++ b/packages/app/modules/pack/hooks/useImportPackItem.ts @@ -7,7 +7,8 @@ export const useImportPackItem = () => { if (!newItem) { throw new Error('Item data is not available.'); } - + }, + onSuccess: (data, newItem, context) => { const previousPack = utils.getPackById.getData({ packId: newItem.packId, }); @@ -32,14 +33,12 @@ export const useImportPackItem = () => { newQueryData as any, ); - return { - previousPack, - }; - }, - onSuccess: () => { utils.getPackById.invalidate(); utils.getPacks.invalidate(); }, + onError: (error, newItem, context) => { + console.error('Error adding item:', error); + }, }); return { diff --git a/packages/ui/src/CascadedDropdown.tsx b/packages/ui/src/CascadedDropdown.tsx new file mode 100644 index 000000000..76a7c56bd --- /dev/null +++ b/packages/ui/src/CascadedDropdown.tsx @@ -0,0 +1,41 @@ +import { RSelect } from '@packrat/ui'; +import React from 'react'; +import { View, Platform } from 'react-native'; + +interface DropdownComponentProps { + width?: string | number; + style?: any; + placeholder?: any; + native?: boolean; + // zeego?: boolean; + [x: string]: any; // for the rest of the props +} + +export const CascadedDropdownComponent: React.FC = ({ + width, + style = {}, + placeholder, + // zeego = false, + ...props +}) => { + const isWeb = Platform.OS === 'web'; + + return ( + + + + ); +}; + +export default CascadedDropdownComponent; diff --git a/packages/ui/src/index.tsx b/packages/ui/src/index.tsx index f7d319802..3d15e888b 100644 --- a/packages/ui/src/index.tsx +++ b/packages/ui/src/index.tsx @@ -33,6 +33,7 @@ export { Container } from './Container'; export { MainContentWeb } from './MainContentWeb'; export { ContextMenu, RContextMenu } from './RContextMenu'; export { DropdownComponent } from './Dropdown'; +export { CascadedDropdownComponent } from './CascadedDropdown'; // export { DropdownMenu, ExampleDropdown } from './RDropdown/DropdownBase'; export { RSkeleton } from './RSkeleton'; diff --git a/server/package.json b/server/package.json index a79b4a52a..c1d8a8da6 100644 --- a/server/package.json +++ b/server/package.json @@ -69,6 +69,7 @@ "celebrate": "^15.0.1", "compression": "^1.7.4", "cors": "^2.8.5", + "crypto-js": "^4.2.0", "csurf": "^1.11.0", "dotenv": "^16.0.3", "drizzle-orm": "^0.30.10", @@ -98,11 +99,13 @@ "openai": "4.25.0", "osmtogeojson": "^3.0.0-beta.5", "p-queue": "^8.0.1", + "papaparse": "^5.4.1", "passport": "^0.6.0", "passport-google-oauth20": "^2.0.0", "passport-local": "^1.0.0", "piscina": "^4.0.0", "prisma": "^5.7.0", + "querystring": "^0.2.1", "radash": "^12.1.0", "superjson": "^2.0.0", "swagger-jsdoc": "^6.2.8", @@ -112,6 +115,7 @@ "trpc-playground": "^1.0.4", "uuid": "^9.0.0", "validator": "^13.9.0", + "xml2js": "^0.6.2", "zod": "^3.22.4" }, "devDependencies": { @@ -120,7 +124,10 @@ "@cloudflare/workers-types": "4.20240419.0", "@faker-js/faker": "^8.4.1", "@openapitools/openapi-generator-cli": "^2.7.0", + "@types/crypto-js": "^4", "@types/jest": "^29.5.11", + "@types/papaparse": "^5", + "@types/xml2js": "^0", "@typescript-eslint/eslint-plugin": "^6.21.0", "@vitest/ui": "^1.6.0", "cross-env": "^7.0.3", diff --git a/server/src/controllers/item/importFromBucket.ts b/server/src/controllers/item/importFromBucket.ts new file mode 100644 index 000000000..30a4a9a33 --- /dev/null +++ b/server/src/controllers/item/importFromBucket.ts @@ -0,0 +1,121 @@ +import { protectedProcedure } from '../../trpc'; +import { + bulkAddItemsGlobalService, + parseCSVData, + listBucketContents, + fetchFromS3, +} from '../../services/item/item.service'; +import { z } from 'zod'; + +export const importFromBucket = async (c) => { + const { directory, ownerId } = await c.req.query(); + + const endpoint = c.env.BUCKET_ENDPOINT; + const bucket = c.env.BUCKET_NAME; + const method = 'GET'; + const region = c.env.BUCKET_REGION; + const service = c.env.BUCKET_SERVICE; + const accessKeyId = c.env.BUCKET_ACCESS_KEY_ID; + const secretKey = c.env.BUCKET_SECRET_KEY; + const sessionToken = c.env.BUCKET_SESSION_TOKEN; + const algorithm = c.env.AWS_SIGN_ALGORITHM; + const x_amz_token = c.env.X_AMZ_SECURITY_TOKEN; + + try { + const latestFileName = await listBucketContents( + endpoint, + bucket, + directory, + method, + service, + region, + accessKeyId, + secretKey, + sessionToken, + algorithm, + x_amz_token, + ); + + if (!latestFileName) { + throw new Error('No files found in the backcountry directory'); + } + + const fileData = await fetchFromS3( + `${endpoint}/${bucket}/${latestFileName}`, + method, + service, + region, + accessKeyId, + secretKey, + sessionToken, + algorithm, + x_amz_token, + ); + + const itemsToInsert = await parseCSVData(fileData, ownerId); + const insertedItems = await bulkAddItemsGlobalService( + itemsToInsert, + c.executionCtx, + ); + + return c.json({ + message: 'Items inserted successfully', + data: insertedItems, + }); + } catch (err) { + console.error('Error:', err); + return c.json({ error: 'An error occurred' }); + } +}; + +export function importFromBucketRoute() { + return protectedProcedure + .input(z.object({ directory: z.string(), ownerId: z.string() })) + .mutation(async (opts) => { + const { directory, ownerId } = opts.input; + const { env, executionCtx }: any = opts.ctx; + + try { + const latestFileName = await listBucketContents( + env.BUCKET_ENDPOINT, + env.BUCKET_NAME, + directory, + 'GET', + env.BUCKET_SERVICE, + env.BUCKET_REGION, + env.BUCKET_ACCESS_KEY_ID, + env.BUCKET_SECRET_KEY, + env.BUCKET_SESSION_TOKEN, + env.AWS_SIGN_ALGORITHM, + env.X_AMZ_SECURITY_TOKEN, + ); + + if (!latestFileName) { + throw new Error('No files found in the directory'); + } + + const fileData = await fetchFromS3( + `${env.BUCKET_ENDPOINT}/${env.BUCKET_NAME}/${latestFileName}`, + 'GET', + env.BUCKET_SERVICE, + env.BUCKET_REGION, + env.BUCKET_ACCESS_KEY_ID, + env.BUCKET_SECRET_KEY, + env.BUCKET_SESSION_TOKEN, + env.AWS_SIGN_ALGORITHM, + env.X_AMZ_SECURITY_TOKEN, + ); + + const itemsToInsert = await parseCSVData(fileData, ownerId); + const insertedItems = await bulkAddItemsGlobalService( + itemsToInsert, + executionCtx, + ); + + return insertedItems; + } catch (err) { + console.error('Error:', err); + throw new Error(`An error occurred: ${err.message}`); + } + }); +} diff --git a/server/src/controllers/item/index.ts b/server/src/controllers/item/index.ts index 143545048..440562579 100644 --- a/server/src/controllers/item/index.ts +++ b/server/src/controllers/item/index.ts @@ -12,3 +12,4 @@ export * from './editGlobalItemAsDuplicate'; export * from './getItemsGlobally'; export * from './searchItemsByName'; export * from './getSimilarItems'; +export * from './importFromBucket'; diff --git a/server/src/drizzle/methods/Item.ts b/server/src/drizzle/methods/Item.ts index b7218b38d..41d3dd80c 100644 --- a/server/src/drizzle/methods/Item.ts +++ b/server/src/drizzle/methods/Item.ts @@ -16,6 +16,23 @@ export class Item { } } + async createBulk(data: InsertItem[]) { + try { + const insertedItems = []; + for (const itemData of data) { + const item = await DbClient.instance + .insert(ItemTable) + .values(itemData) + .returning() + .get(); + insertedItems.push(item); + } + return insertedItems; + } catch (error) { + throw new Error(`Failed to create items: ${error.message}`); + } + } + async update( id: string, data: Partial, diff --git a/server/src/routes/index.ts b/server/src/routes/index.ts index 38944ac30..5d59d5f53 100644 --- a/server/src/routes/index.ts +++ b/server/src/routes/index.ts @@ -1,5 +1,3 @@ -// import path from 'path'; -// import csrf from 'csurf'; import packRoutes from './packRoutes'; import itemRoutes from './itemRoutes'; import tripRoutes from './tripRoutes'; @@ -13,37 +11,15 @@ import openAiRoutes from './openAiRoutes'; import templateRoutes from './templateRoutes'; import favoriteRouters from './favoriteRoutes'; import userRoutes from './userRoutes'; +import testRoutes from './testRoutes'; import mapPreviewRouter from './mapPreviewRouter'; import healthRoutes from './healthRoutes'; import { Hono } from 'hono'; const router = new Hono(); -// Create a CSRF middleware -// const csrfProtection = csrf({ cookie: true }); - -// /** -// * Logs the incoming request method and path, and logs the finished request method, path, status code, and request body. -// * -// * @param {Request} req - The incoming request object. -// * @param {Response} res - The response object. -// * @param {NextFunction} next - The next function to call in the middleware chain. -// */ -// const logger = (req: Request, res: Response, next: express.NextFunction) => { -// console.log(`Incoming ${req.method} ${req.path}`); -// res.on('finish', () => { -// console.log(`Finished ${req.method} ${req.path} ${res.statusCode}`); -// console.log(`Body ${req.body}`); -// }); -// next(); -// }; - -// // use logger middleware in development -// if (process.env.NODE_ENV !== 'production') { -// router.use(logger); -// } - // use routes +router.route('/testapi', testRoutes); router.route('/user', userRoutes); router.route('/pack', packRoutes); router.route('/item', itemRoutes); @@ -60,13 +36,8 @@ router.route('/favorite', favoriteRouters); router.route('/mapPreview', mapPreviewRouter); router.route('/health', healthRoutes); -const helloRouter = new Hono(); -helloRouter.get('/', (c) => { - return c.text('Hello, world!'); -}); -router.route('/hello', helloRouter); - // Also listen to /api for backwards compatibility +router.route('/api/testapi', testRoutes); router.route('/api/user', userRoutes); router.route('/api/pack', packRoutes); router.route('/api/item', itemRoutes); @@ -83,38 +54,4 @@ router.route('/api/favorite', favoriteRouters); router.route('/api/openai', openAiRoutes); router.route('/api/mapPreview', mapPreviewRouter); -// // Static routes for serving the React Native Web app -// if (process.env.NODE_ENV === 'production') { -// const __dirname = path.resolve(); -// const serverType = process.env.REACT_APP_SERVER_TYPE || 'vite'; - -// // Serve the client's index.html file at the root route -// router.get('/', (req, res) => { -// // Attach the CSRF token cookie to the response -// // res.cookie("XSRF-TOKEN", req.csrfToken()); - -// const basePath = serverType === 'next' ? '../apps/next/out' : '../apps/vite/dist'; -// res.sendFile(path.resolve(__dirname, basePath, 'index.html')); -// }); - -// // Serve the static assets -// const staticPath = serverType === 'next' ? '../apps/next/out' : '../apps/vite/dist'; -// router.use(express.static(path.join(__dirname, staticPath))); - -// // Serve the client's index.html file at all other routes NOT starting with /api -// router.get(/^(?!\/?api).*/, (req, res) => { -// // res.cookie("XSRF-TOKEN", req.csrfToken()); -// const basePath = serverType === 'next' ? '../apps/next/out' : '../apps/vite/dist'; -// res.sendFile(path.resolve(__dirname, basePath, 'index.html')); -// }); -// } - -// // Attach the CSRF token to a specific route in development -// if (process.env.NODE_ENV !== 'production') { -// router.get('/api/csrf/restore', (req, res) => { -// // res.cookie("XSRF-TOKEN", req.csrfToken()); -// res.status(201).json({}); -// }); -// } - export default router; diff --git a/server/src/routes/itemRoutes.ts b/server/src/routes/itemRoutes.ts index bf458dc3a..94253f700 100644 --- a/server/src/routes/itemRoutes.ts +++ b/server/src/routes/itemRoutes.ts @@ -14,6 +14,7 @@ import { importItems, importItemsGlobal, getSimilarItems, + importFromBucket, } from '../controllers/item/index'; import * as validator from '@packrat/validations'; import { tryCatchWrapper } from '../helpers/tryCatchWrapper'; @@ -37,6 +38,13 @@ router.get( tryCatchWrapper(getSimilarItems), ); +router.get( + '/importFromBucket', + authTokenMiddleware, + // zodParser(validator.importFromBucket, 'query'), + tryCatchWrapper(importFromBucket), +); + router.get( '/i/:id', authTokenMiddleware, diff --git a/server/src/routes/testRoutes.ts b/server/src/routes/testRoutes.ts new file mode 100644 index 000000000..9a073ba7b --- /dev/null +++ b/server/src/routes/testRoutes.ts @@ -0,0 +1,52 @@ +import { Hono } from 'hono'; +import querystring from 'querystring'; + +const router = new Hono(); + +router.get('/', async (c) => { + const params = c.req.query(); + console.log('Received data:', params); + + return c.json({ message: 'Data received successfully!', data: params }); +}); + +router.get('/test', async (c) => { + try { + const postData = querystring.stringify({ + project: 'PackRat', + spider: 'backcountry', + }); + + const response = await fetch('http://localhost:6800/schedule.json', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: postData, + }); + + const responseData = await response.json(); + + if (responseData.status === 'ok') { + console.log('Scraping initiated', responseData); + return c.json({ + message: 'Scraping initiated successfully!', + response: responseData, + }); + } else { + console.error('Error from Scrapyd:', responseData); + return c.json({ + message: 'Failed to initiate scraping', + error: responseData, + }); + } + } catch (error) { + console.error('Error initiating scraping:', error); + return c.json({ + message: 'Failed to initiate scraping', + error: error.toString(), + }); + } +}); + +export default router; diff --git a/server/src/routes/trpcRouter.ts b/server/src/routes/trpcRouter.ts index 78c13c45b..e399ec51c 100644 --- a/server/src/routes/trpcRouter.ts +++ b/server/src/routes/trpcRouter.ts @@ -71,6 +71,7 @@ import { getItemsRoute, searchItemsByNameRoute, getSimilarItemsRoute, + importFromBucketRoute, } from '../controllers/item'; import { getTrailsRoute } from '../controllers/getTrail'; import { getParksRoute } from '../controllers/getParks'; @@ -161,6 +162,7 @@ export const appRouter = trpcRouter({ editGlobalItemAsDuplicate: editGlobalItemAsDuplicateRoute(), // Not Implemented deleteGlobalItem: deleteGlobalItemRoute(), // Done, getSimilarItems: getSimilarItemsRoute(), + importFromBucket: importFromBucketRoute(), // trails routes getTrails: getTrailsRoute(), // // parks route diff --git a/server/src/services/item/bulkAddGlobalItemService.ts b/server/src/services/item/bulkAddGlobalItemService.ts new file mode 100644 index 000000000..b7733d724 --- /dev/null +++ b/server/src/services/item/bulkAddGlobalItemService.ts @@ -0,0 +1,56 @@ +import { type ExecutionContext } from 'hono'; +import { type InsertItemCategory } from '../../db/schema'; +import { ItemCategory } from '../../drizzle/methods/itemcategory'; +import { DbClient } from 'src/db/client'; +import { item as ItemTable } from '../../db/schema'; + +export const bulkAddItemsGlobalService = async ( + items: Array<{ + name: string; + weight: number; + quantity: number; + unit: string; + type: 'Food' | 'Water' | 'Essentials'; + ownerId: string; + }>, + executionCtx: ExecutionContext, +) => { + const categories = ['Food', 'Water', 'Essentials']; + + const itemCategoryClass = new ItemCategory(); + const insertedItems = []; + + for (const itemData of items) { + const { name, weight, quantity, unit, type, ownerId } = itemData; + if (!categories.includes(type)) { + throw new Error(`Category must be one of: ${categories.join(', ')}`); + } + + let category: InsertItemCategory | null; + category = + (await itemCategoryClass.findItemCategory({ name: type })) || null; + if (!category) { + category = await itemCategoryClass.create({ name: type }); + } + + const newItem = { + name, + weight, + quantity, + unit, + categoryId: category.id, + global: true, + ownerId, + }; + + const item = await DbClient.instance + .insert(ItemTable) + .values(newItem) + .returning() + .get(); + + insertedItems.push(item); + } + + return insertedItems; +}; diff --git a/server/src/services/item/importFromBucketService.ts b/server/src/services/item/importFromBucketService.ts new file mode 100644 index 000000000..7468b47aa --- /dev/null +++ b/server/src/services/item/importFromBucketService.ts @@ -0,0 +1,186 @@ +import * as CryptoJS from 'crypto-js'; +import { parseStringPromise } from 'xml2js'; +import Papa from 'papaparse'; + +interface CSVType { + name: string; + weight: number; + quantity: number; + unit: string; + type: 'Essentials' | 'Food' | 'Water'; + ownerId: string; +} + +function getSignatureKey( + key: string, + dateStamp: string, + regionName: string, + serviceName: string, +) { + const kDate = CryptoJS.HmacSHA256(dateStamp, 'AWS4' + key); + const kRegion = CryptoJS.HmacSHA256(regionName, kDate); + const kService = CryptoJS.HmacSHA256(serviceName, kRegion); + const kSigning = CryptoJS.HmacSHA256('aws4_request', kService); + return kSigning; +} + +function generateAWSHeaders( + url: string, + method: string, + service: string, + region: string, + accessKey: string, + secretKey: string, + sessionToken: string, + algorithm: string, + x_amz_token: string, +) { + const amzDate = new Date() + .toISOString() + .replace(/[:-]/g, '') + .replace(/\.\d{3}/, ''); + const dateStamp = amzDate.slice(0, 8); + const canonicalUri = new URL(url).pathname; + const canonicalQueryString = ''; + const payloadHash = CryptoJS.SHA256('').toString(CryptoJS.enc.Hex); + const canonicalHeaders = + `host:${new URL(url).hostname}\nx-amz-date:${amzDate}\n` + + (sessionToken ? `x-amz-security-token:${sessionToken}\n` : ''); + const signedHeaders = + 'host;x-amz-date' + (sessionToken ? `;${x_amz_token}` : ''); + const canonicalRequest = `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}`; + + const credentialScope = `${dateStamp}/${region}/${service}/aws4_request`; + const stringToSign = `${algorithm}\n${amzDate}\n${credentialScope}\n${CryptoJS.SHA256(canonicalRequest).toString(CryptoJS.enc.Hex)}`; + + const signingKey = getSignatureKey(secretKey, dateStamp, region, service); + const signature = CryptoJS.HmacSHA256(stringToSign, signingKey).toString( + CryptoJS.enc.Hex, + ); + const authorizationHeader = `${algorithm} Credential=${accessKey}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`; + + return { + host: new URL(url).hostname, + 'x-amz-date': amzDate, + 'x-amz-content-sha256': payloadHash, + Authorization: authorizationHeader, + ...(sessionToken && { 'x-amz-security-token': sessionToken }), + }; +} + +export async function fetchFromS3( + url: string, + method: string, + service: string, + region: string, + accessKey: string, + secretKey: string, + sessionToken: string, + algorithm: string, + x_amz_token: string, +) { + const headers = generateAWSHeaders( + url, + method, + service, + region, + accessKey, + secretKey, + sessionToken, + algorithm, + x_amz_token, + ); + + const response = await fetch(url, { + method, + headers, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch from S3: ${response.statusText}`); + } + + const data = await response.text(); + return data; +} + +export async function listBucketContents( + endpoint: string, + bucket: string, + directory: string, + method: string, + service: string, + region: string, + accessKey: string, + secretKey: string, + sessionToken: string, + algorithm: string, + x_amz_token: string, +) { + const listData = await fetchFromS3( + `${endpoint}/${bucket}`, + method, + service, + region, + accessKey, + secretKey, + sessionToken, + algorithm, + x_amz_token, + ); + + const bucketOptions = [ + { label: 'bucket 1', value: 'rei' }, + { label: 'bucket 2', value: 'sierra' }, + { label: 'bucket 3', value: 'cabelas' }, + { label: 'bucket 4', value: 'moosejaw' }, + { label: 'bucket 5', value: 'backcountry' }, + ]; + + directory = + bucketOptions.find((item) => item.label === directory)?.value || ''; + + const parsedListData = await parseStringPromise(listData); + const contents = parsedListData.ListBucketResult.Contents; + + const fileNames = contents + .filter((item) => item.Key[0].startsWith(`${directory}/`)) + .map((item) => item.Key[0]); + + return fileNames.sort().reverse()[0]; // Get the latest file name +} + +export async function parseCSVData(fileData: string, ownerId: string) { + return new Promise((resolve, reject) => { + const itemsToInsert = []; + + Papa.parse(fileData, { + header: true, + complete: (results) => { + try { + for (const [index, item] of results.data.entries()) { + if ( + index === results.data.length - 1 && + Object.values(item).every((value) => value === '') + ) { + continue; + } + + itemsToInsert.push({ + name: item.name, + weight: item.claimed_weight || 0, + quantity: item.quantity || 1, + unit: item.claimed_weight_unit || 'g', + type: 'Essentials', + ownerId, + }); + } + resolve(itemsToInsert); + } catch (error) { + reject(error); + } + }, + error: (error) => reject(error), + }); + }); +} diff --git a/server/src/services/item/item.service.ts b/server/src/services/item/item.service.ts index 1ca7d6af7..fb24b7bc3 100644 --- a/server/src/services/item/item.service.ts +++ b/server/src/services/item/item.service.ts @@ -9,3 +9,5 @@ export * from './getItemByIdService'; export * from './getItemsGloballyService'; export * from './searchItemsByNameService'; export * from './getItemsService'; +export * from './bulkAddGlobalItemService'; +export * from './importFromBucketService'; diff --git a/server/vitest.config.d.ts b/server/vitest.config.d.ts index 06c214f2f..85bcff909 100644 --- a/server/vitest.config.d.ts +++ b/server/vitest.config.d.ts @@ -1,4 +1,2 @@ -declare const _default: import('@cloudflare/vitest-pool-workers/config').AnyConfigExport< - import('@cloudflare/vitest-pool-workers/config').WorkersProjectConfigExport ->; +declare const _default: import("@cloudflare/vitest-pool-workers/config").AnyConfigExport; export default _default; diff --git a/yarn.lock b/yarn.lock index 23dc2a1ad..d748bbabb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11494,6 +11494,13 @@ __metadata: languageName: node linkType: hard +"@types/crypto-js@npm:^4": + version: 4.2.2 + resolution: "@types/crypto-js@npm:4.2.2" + checksum: 10/a40fc5a9219fd33f54ba3e094c5e5ab2904d3106681a76f1029bb038976591e9c8959099963bf4474fde21c2d8d00c1f896445206a3a58f85588f9cb1bd96a9a + languageName: node + linkType: hard + "@types/csurf@npm:*, @types/csurf@npm:^1.11.2": version: 1.11.5 resolution: "@types/csurf@npm:1.11.5" @@ -12114,6 +12121,15 @@ __metadata: languageName: node linkType: hard +"@types/xml2js@npm:^0": + version: 0.4.14 + resolution: "@types/xml2js@npm:0.4.14" + dependencies: + "@types/node": "npm:*" + checksum: 10/d76338b8d6ce8540c7af6a32aacf96c38f6de48254568f58f6e5ac2af3f88e6bd1490e5346d3bb336990f91267d23c5cc09e8bf7e80840a63c7855dbf174ecbb + languageName: node + linkType: hard + "@types/yargs-parser@npm:*": version: 21.0.3 resolution: "@types/yargs-parser@npm:21.0.3" @@ -16913,6 +16929,13 @@ __metadata: languageName: node linkType: hard +"crypto-js@npm:^4.2.0": + version: 4.2.0 + resolution: "crypto-js@npm:4.2.0" + checksum: 10/c7bcc56a6e01c3c397e95aa4a74e4241321f04677f9a618a8f48a63b5781617248afb9adb0629824792e7ec20ca0d4241a49b6b2938ae6f973ec4efc5c53c924 + languageName: node + linkType: hard + "crypto-random-string@npm:^1.0.0": version: 1.0.0 resolution: "crypto-random-string@npm:1.0.0" @@ -31233,6 +31256,13 @@ __metadata: languageName: node linkType: hard +"querystring@npm:^0.2.1": + version: 0.2.1 + resolution: "querystring@npm:0.2.1" + checksum: 10/5ae2eeb8c6d70263a3d13ffaf234ce9593ae0e95ad8ea04aa540e14ff66679347420817aeb4fe6fdfa2aaa7fac86e311b6f1d3da2187f433082ad9125c808c14 + languageName: node + linkType: hard + "queue-microtask@npm:^1.2.2": version: 1.2.3 resolution: "queue-microtask@npm:1.2.3" @@ -33388,12 +33418,15 @@ __metadata: "@trpc/server": "npm:^10.45.1" "@types/bcrypt": "npm:^5.0.0" "@types/cors": "npm:^2.8.13" + "@types/crypto-js": "npm:^4" "@types/csurf": "npm:^1.11.2" "@types/jest": "npm:^29.5.11" "@types/node": "npm:^20.14.2" + "@types/papaparse": "npm:^5" "@types/swagger-jsdoc": "npm:^6.0.1" "@types/swagger-ui-express": "npm:^4.1.3" "@types/validator": "npm:^13.11.1" + "@types/xml2js": "npm:^0" "@typescript-eslint/eslint-plugin": "npm:^6.21.0" "@typescript-eslint/parser": "npm:^6.21.0" "@vitest/ui": "npm:^1.6.0" @@ -33406,6 +33439,7 @@ __metadata: compression: "npm:^1.7.4" cors: "npm:^2.8.5" cross-env: "npm:^7.0.3" + crypto-js: "npm:^4.2.0" csurf: "npm:^1.11.0" dotenv: "npm:^16.0.3" drizzle-kit: "npm:^0.20.17" @@ -33449,12 +33483,14 @@ __metadata: openapi-generator: "npm:^0.1.39" osmtogeojson: "npm:^3.0.0-beta.5" p-queue: "npm:^8.0.1" + papaparse: "npm:^5.4.1" passport: "npm:^0.6.0" passport-google-oauth20: "npm:^2.0.0" passport-local: "npm:^1.0.0" piscina: "npm:^4.0.0" prettier: "npm:^3.2.5" prisma: "npm:^5.7.0" + querystring: "npm:^0.2.1" radash: "npm:^12.1.0" superjson: "npm:^2.0.0" swagger-jsdoc: "npm:^6.2.8" @@ -33470,6 +33506,7 @@ __metadata: validator: "npm:^13.9.0" vitest: "npm:1.3.0" wrangler: "npm:^3.51.2" + xml2js: "npm:^0.6.2" zod: "npm:^3.22.4" languageName: unknown linkType: soft @@ -37821,7 +37858,7 @@ __metadata: languageName: node linkType: hard -"xml2js@npm:0.6.2": +"xml2js@npm:0.6.2, xml2js@npm:^0.6.2": version: 0.6.2 resolution: "xml2js@npm:0.6.2" dependencies: