From 55bc9d1934927ec17bdc3ca7e0f64786a692fb4a Mon Sep 17 00:00:00 2001 From: pinocchio-life-like Date: Thu, 15 Aug 2024 09:20:41 +0300 Subject: [PATCH] cleanup --- apps/vite/src/routeTree.gen.ts | 177 +--------- .../src/controllers/item/importFromBucket.ts | 314 +++--------------- .../services/item/importFromBucketService.ts | 175 ++++++++++ server/src/services/item/item.service.ts | 1 + 4 files changed, 227 insertions(+), 440 deletions(-) create mode 100644 server/src/services/item/importFromBucketService.ts diff --git a/apps/vite/src/routeTree.gen.ts b/apps/vite/src/routeTree.gen.ts index 8e9dfcd1e..968fb50c5 100644 --- a/apps/vite/src/routeTree.gen.ts +++ b/apps/vite/src/routeTree.gen.ts @@ -174,163 +174,94 @@ const ProfileSettingsIndexLazyRoute = ProfileSettingsIndexLazyImport.update({ declare module '@tanstack/react-router' { interface FileRoutesByPath { '/': { - id: '/' - path: '/' - fullPath: '/' preLoaderRoute: typeof IndexImport parentRoute: typeof rootRoute } '/destination/query': { - id: '/destination/query' - path: '/destination/query' - fullPath: '/destination/query' preLoaderRoute: typeof DestinationQueryLazyImport parentRoute: typeof rootRoute } '/item/$itemId': { - id: '/item/$itemId' - path: '/item/$itemId' - fullPath: '/item/$itemId' preLoaderRoute: typeof ItemItemIdLazyImport parentRoute: typeof rootRoute } '/pack/$id': { - id: '/pack/$id' - path: '/pack/$id' - fullPath: '/pack/$id' preLoaderRoute: typeof PackIdLazyImport parentRoute: typeof rootRoute } '/pack/create': { - id: '/pack/create' - path: '/pack/create' - fullPath: '/pack/create' preLoaderRoute: typeof PackCreateLazyImport parentRoute: typeof rootRoute } '/profile/$id': { - id: '/profile/$id' - path: '/profile/$id' - fullPath: '/profile/$id' preLoaderRoute: typeof ProfileIdLazyImport parentRoute: typeof rootRoute } '/trip/$tripId': { - id: '/trip/$tripId' - path: '/trip/$tripId' - fullPath: '/trip/$tripId' preLoaderRoute: typeof TripTripIdLazyImport parentRoute: typeof rootRoute } '/trip/create': { - id: '/trip/create' - path: '/trip/create' - fullPath: '/trip/create' preLoaderRoute: typeof TripCreateLazyImport parentRoute: typeof rootRoute } '/about/': { - id: '/about/' - path: '/about' - fullPath: '/about' preLoaderRoute: typeof AboutIndexLazyImport parentRoute: typeof rootRoute } '/appearance/': { - id: '/appearance/' - path: '/appearance' - fullPath: '/appearance' preLoaderRoute: typeof AppearanceIndexLazyImport parentRoute: typeof rootRoute } '/dashboard/': { - id: '/dashboard/' - path: '/dashboard' - fullPath: '/dashboard' preLoaderRoute: typeof DashboardIndexLazyImport parentRoute: typeof rootRoute } '/feed/': { - id: '/feed/' - path: '/feed' - fullPath: '/feed' preLoaderRoute: typeof FeedIndexLazyImport parentRoute: typeof rootRoute } '/items/': { - id: '/items/' - path: '/items' - fullPath: '/items' preLoaderRoute: typeof ItemsIndexLazyImport parentRoute: typeof rootRoute } '/map/': { - id: '/map/' - path: '/map' - fullPath: '/map' preLoaderRoute: typeof MapIndexLazyImport parentRoute: typeof rootRoute } '/maps/': { - id: '/maps/' - path: '/maps' - fullPath: '/maps' preLoaderRoute: typeof MapsIndexLazyImport parentRoute: typeof rootRoute } '/packs/': { - id: '/packs/' - path: '/packs' - fullPath: '/packs' preLoaderRoute: typeof PacksIndexLazyImport parentRoute: typeof rootRoute } '/password-reset/': { - id: '/password-reset/' - path: '/password-reset' - fullPath: '/password-reset' preLoaderRoute: typeof PasswordResetIndexLazyImport parentRoute: typeof rootRoute } '/privacy/': { - id: '/privacy/' - path: '/privacy' - fullPath: '/privacy' preLoaderRoute: typeof PrivacyIndexLazyImport parentRoute: typeof rootRoute } '/profile/': { - id: '/profile/' - path: '/profile' - fullPath: '/profile' preLoaderRoute: typeof ProfileIndexLazyImport parentRoute: typeof rootRoute } '/register/': { - id: '/register/' - path: '/register' - fullPath: '/register' preLoaderRoute: typeof RegisterIndexLazyImport parentRoute: typeof rootRoute } '/sign-in/': { - id: '/sign-in/' - path: '/sign-in' - fullPath: '/sign-in' preLoaderRoute: typeof SignInIndexLazyImport parentRoute: typeof rootRoute } '/trips/': { - id: '/trips/' - path: '/trips' - fullPath: '/trips' preLoaderRoute: typeof TripsIndexLazyImport parentRoute: typeof rootRoute } '/profile/settings/': { - id: '/profile/settings/' - path: '/profile/settings' - fullPath: '/profile/settings' preLoaderRoute: typeof ProfileSettingsIndexLazyImport parentRoute: typeof rootRoute } @@ -339,7 +270,7 @@ declare module '@tanstack/react-router' { // Create and export the route tree -export const routeTree = rootRoute.addChildren({ +export const routeTree = rootRoute.addChildren([ IndexRoute, DestinationQueryLazyRoute, ItemItemIdLazyRoute, @@ -363,110 +294,6 @@ export const routeTree = rootRoute.addChildren({ SignInIndexLazyRoute, TripsIndexLazyRoute, ProfileSettingsIndexLazyRoute, -}) +]) /* prettier-ignore-end */ - -/* ROUTE_MANIFEST_START -{ - "routes": { - "__root__": { - "filePath": "__root.tsx", - "children": [ - "/", - "/destination/query", - "/item/$itemId", - "/pack/$id", - "/pack/create", - "/profile/$id", - "/trip/$tripId", - "/trip/create", - "/about/", - "/appearance/", - "/dashboard/", - "/feed/", - "/items/", - "/map/", - "/maps/", - "/packs/", - "/password-reset/", - "/privacy/", - "/profile/", - "/register/", - "/sign-in/", - "/trips/", - "/profile/settings/" - ] - }, - "/": { - "filePath": "index.tsx" - }, - "/destination/query": { - "filePath": "destination/query.lazy.tsx" - }, - "/item/$itemId": { - "filePath": "item/$itemId.lazy.tsx" - }, - "/pack/$id": { - "filePath": "pack/$id.lazy.tsx" - }, - "/pack/create": { - "filePath": "pack/create.lazy.tsx" - }, - "/profile/$id": { - "filePath": "profile/$id.lazy.tsx" - }, - "/trip/$tripId": { - "filePath": "trip/$tripId.lazy.tsx" - }, - "/trip/create": { - "filePath": "trip/create.lazy.tsx" - }, - "/about/": { - "filePath": "about/index.lazy.tsx" - }, - "/appearance/": { - "filePath": "appearance/index.lazy.tsx" - }, - "/dashboard/": { - "filePath": "dashboard/index.lazy.tsx" - }, - "/feed/": { - "filePath": "feed/index.lazy.tsx" - }, - "/items/": { - "filePath": "items/index.lazy.tsx" - }, - "/map/": { - "filePath": "map/index.lazy.tsx" - }, - "/maps/": { - "filePath": "maps/index.lazy.tsx" - }, - "/packs/": { - "filePath": "packs/index.lazy.tsx" - }, - "/password-reset/": { - "filePath": "password-reset/index.lazy.tsx" - }, - "/privacy/": { - "filePath": "privacy/index.lazy.tsx" - }, - "/profile/": { - "filePath": "profile/index.lazy.tsx" - }, - "/register/": { - "filePath": "register/index.lazy.tsx" - }, - "/sign-in/": { - "filePath": "sign-in/index.lazy.tsx" - }, - "/trips/": { - "filePath": "trips/index.lazy.tsx" - }, - "/profile/settings/": { - "filePath": "profile/settings/index.lazy.tsx" - } - } -} -ROUTE_MANIFEST_END */ diff --git a/server/src/controllers/item/importFromBucket.ts b/server/src/controllers/item/importFromBucket.ts index 748d401d8..30a4a9a33 100644 --- a/server/src/controllers/item/importFromBucket.ts +++ b/server/src/controllers/item/importFromBucket.ts @@ -1,71 +1,12 @@ import { protectedProcedure } from '../../trpc'; -import { bulkAddItemsGlobalService } from '../../services/item/item.service'; -import * as CryptoJS from 'crypto-js'; -import { parseStringPromise } from 'xml2js'; -import Papa from 'papaparse'; +import { + bulkAddItemsGlobalService, + parseCSVData, + listBucketContents, + fetchFromS3, +} from '../../services/item/item.service'; import { z } from 'zod'; -interface CSVType { - name: string; - claimed_weight: number; - quantity: number; - claimed_weight_unit: string; - type: string; - ownerId: string; -} - -function getSignatureKey(key, dateStamp, regionName, serviceName) { - const kDate = CryptoJS.HmacSHA256(dateStamp, 'AWS4' + key); - const kRegion = CryptoJS.HmacSHA256(regionName, kDate); - const kService = CryptoJS.HmacSHA256(serviceName, kRegion); - const kSigning = CryptoJS.HmacSHA256('aws4_request', kService); - return kSigning; -} - -function generateAWSHeaders( - url, - method, - service, - region, - accessKey, - secretKey, - sessionToken, - algorithm, - x_amz_token, -) { - const amzDate = new Date() - .toISOString() - .replace(/[:-]/g, '') - .replace(/\.\d{3}/, ''); - const dateStamp = amzDate.slice(0, 8); - const canonicalUri = new URL(url).pathname; - const canonicalQueryString = ''; - const payloadHash = CryptoJS.SHA256('').toString(CryptoJS.enc.Hex); - const canonicalHeaders = - `host:${new URL(url).hostname}\nx-amz-date:${amzDate}\n` + - (sessionToken ? `x-amz-security-token:${sessionToken}\n` : ''); - const signedHeaders = - 'host;x-amz-date' + (sessionToken ? `;${x_amz_token}` : ''); - const canonicalRequest = `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}`; - - const credentialScope = `${dateStamp}/${region}/${service}/aws4_request`; - const stringToSign = `${algorithm}\n${amzDate}\n${credentialScope}\n${CryptoJS.SHA256(canonicalRequest).toString(CryptoJS.enc.Hex)}`; - - const signingKey = getSignatureKey(secretKey, dateStamp, region, service); - const signature = CryptoJS.HmacSHA256(stringToSign, signingKey).toString( - CryptoJS.enc.Hex, - ); - const authorizationHeader = `${algorithm} Credential=${accessKey}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`; - - return { - host: new URL(url).hostname, - 'x-amz-date': amzDate, - 'x-amz-content-sha256': payloadHash, - Authorization: authorizationHeader, - ...(sessionToken && { 'x-amz-security-token': sessionToken }), - }; -} - export const importFromBucket = async (c) => { const { directory, ownerId } = await c.req.query(); @@ -80,45 +21,26 @@ export const importFromBucket = async (c) => { const algorithm = c.env.AWS_SIGN_ALGORITHM; const x_amz_token = c.env.X_AMZ_SECURITY_TOKEN; - // Generate AWS Headers for listing bucket contents - const listHeaders = generateAWSHeaders( - endpoint + '/' + bucket, - method, - service, - region, - accessKeyId, - secretKey, - sessionToken, - algorithm, - x_amz_token, - ); - try { - // Fetch data from bucket to list contents - const listResponse = await fetch(`${endpoint}/${bucket}`, { + const latestFileName = await listBucketContents( + endpoint, + bucket, + directory, method, - headers: listHeaders, - }); - const listData = await listResponse.text(); - - // Parse XML response - const parsedListData = await parseStringPromise(listData); - const contents = parsedListData.ListBucketResult.Contents; - - // Extract and log file names - const fileNames = contents - .filter((item) => item.Key[0].startsWith(`${directory}/`)) - .map((item) => item.Key[0]); - - // Sort file names to get the latest one - const latestFileName = fileNames.sort().reverse()[0]; + service, + region, + accessKeyId, + secretKey, + sessionToken, + algorithm, + x_amz_token, + ); if (!latestFileName) { throw new Error('No files found in the backcountry directory'); } - // Generate AWS Headers for fetching the latest file - const fileHeaders = generateAWSHeaders( + const fileData = await fetchFromS3( `${endpoint}/${bucket}/${latestFileName}`, method, service, @@ -130,63 +52,12 @@ export const importFromBucket = async (c) => { x_amz_token, ); - // Fetch the specific CSV file - const fileResponse = await fetch( - `${endpoint}/${bucket}/${latestFileName}`, - { - method, - headers: fileHeaders, - }, - ); - const fileData = await fileResponse.text(); - - // Check for errors in the file response - if (fileResponse.status !== 200) { - console.error('Error fetching file:', fileData); - return c.json({ error: 'Error fetching file', details: fileData }); - } - - const itemsToInsert = []; - - await new Promise((resolve, reject) => { - Papa.parse(fileData, { - header: true, - complete: (results) => { - try { - for (const [index, item] of results.data.entries()) { - if ( - index === results.data.length - 1 && - Object.values(item).every((value) => value === '') - ) { - continue; - } - - itemsToInsert.push({ - name: item.name, - weight: item.claimed_weight || 0, - quantity: item.quantity || 1, - unit: item.claimed_weight_unit || 'g', - type: 'Essentials', - ownerId, - }); - } - resolve(null); - } catch (error) { - console.error('Error processing CSV data:', error); - reject(error); - } - }, - error: (error) => { - console.error('Error parsing CSV file:', error); - reject(error); - }, - }); - }); - + const itemsToInsert = await parseCSVData(fileData, ownerId); const insertedItems = await bulkAddItemsGlobalService( itemsToInsert, c.executionCtx, ); + return c.json({ message: 'Items inserted successfully', data: insertedItems, @@ -204,131 +75,44 @@ export function importFromBucketRoute() { const { directory, ownerId } = opts.input; const { env, executionCtx }: any = opts.ctx; - const endpoint = env.BUCKET_ENDPOINT; - const bucket = env.BUCKET_NAME; - const method = 'GET'; - const region = env.BUCKET_REGION; - const service = env.BUCKET_SERVICE; - const accessKeyId = env.BUCKET_ACCESS_KEY_ID; - const secretKey = env.BUCKET_SECRET_KEY; - const sessionToken = env.BUCKET_SESSION_TOKEN; - const algorithm = env.AWS_SIGN_ALGORITHM; - const x_amz_token = env.X_AMZ_SECURITY_TOKEN; - - // Generate AWS Headers for listing bucket contents - const listHeaders = generateAWSHeaders( - `${endpoint}/${bucket}`, - method, - service, - region, - accessKeyId, - secretKey, - sessionToken, - algorithm, - x_amz_token, - ); - try { - const listResponse = await fetch(`${endpoint}/${bucket}`, { - method, - headers: listHeaders, - }); - - if (!listResponse.ok) { - throw new Error('Failed to list bucket contents'); - } - - const listData = await listResponse.text(); - - // Parse XML response - const parsedListData = await parseStringPromise(listData); - const contents = parsedListData.ListBucketResult.Contents; - - // Extract and log file names - const fileNames = contents - .filter((item) => item.Key[0].startsWith(`${directory}/`)) - .map((item) => item.Key[0]); - - // Sort file names to get the latest one - const latestFileName = fileNames.sort().reverse()[0]; + const latestFileName = await listBucketContents( + env.BUCKET_ENDPOINT, + env.BUCKET_NAME, + directory, + 'GET', + env.BUCKET_SERVICE, + env.BUCKET_REGION, + env.BUCKET_ACCESS_KEY_ID, + env.BUCKET_SECRET_KEY, + env.BUCKET_SESSION_TOKEN, + env.AWS_SIGN_ALGORITHM, + env.X_AMZ_SECURITY_TOKEN, + ); if (!latestFileName) { throw new Error('No files found in the directory'); } - // Generate AWS Headers for fetching the latest file - const fileHeaders = generateAWSHeaders( - `${endpoint}/${bucket}/${latestFileName}`, - method, - service, - region, - accessKeyId, - secretKey, - sessionToken, - algorithm, - x_amz_token, + const fileData = await fetchFromS3( + `${env.BUCKET_ENDPOINT}/${env.BUCKET_NAME}/${latestFileName}`, + 'GET', + env.BUCKET_SERVICE, + env.BUCKET_REGION, + env.BUCKET_ACCESS_KEY_ID, + env.BUCKET_SECRET_KEY, + env.BUCKET_SESSION_TOKEN, + env.AWS_SIGN_ALGORITHM, + env.X_AMZ_SECURITY_TOKEN, ); - // Fetch the specific CSV file - const fileResponse = await fetch( - `${endpoint}/${bucket}/${latestFileName}`, - { - method, - headers: fileHeaders, - }, + const itemsToInsert = await parseCSVData(fileData, ownerId); + const insertedItems = await bulkAddItemsGlobalService( + itemsToInsert, + executionCtx, ); - if (!fileResponse.ok) { - throw new Error('Failed to fetch the latest file'); - } - - const fileData = await fileResponse.text(); - - return new Promise((resolve, reject) => { - Papa.parse(fileData, { - header: true, - complete: async function (results) { - try { - const itemsToInsert = []; - - for (const [index, item] of results.data.entries()) { - if ( - index === results.data.length - 1 && - Object.values(item).every((value) => value === '') - ) { - continue; - } - - itemsToInsert.push({ - name: item.name, - weight: item.claimed_weight || 0, - quantity: item.quantity || 1, - unit: item.claimed_weight_unit || 'g', - type: 'Essentials', - ownerId, - }); - } - - const insertedItems = await bulkAddItemsGlobalService( - itemsToInsert, - executionCtx, - ); - - return resolve(insertedItems); - } catch (error) { - console.error('Error in bulkAddItemsGlobalService:', error); - - return reject( - new Error(`Failed to add items: ${error.message}`), - ); - } - }, - error: function (error) { - console.error('Error parsing CSV file:', error); - reject(error); - }, - }); - }); + return insertedItems; } catch (err) { console.error('Error:', err); throw new Error(`An error occurred: ${err.message}`); diff --git a/server/src/services/item/importFromBucketService.ts b/server/src/services/item/importFromBucketService.ts new file mode 100644 index 000000000..e911bb880 --- /dev/null +++ b/server/src/services/item/importFromBucketService.ts @@ -0,0 +1,175 @@ +import * as CryptoJS from 'crypto-js'; +import { parseStringPromise } from 'xml2js'; +import Papa from 'papaparse'; + +interface CSVType { + name: string; + claimed_weight: number; + quantity: number; + claimed_weight_unit: string; + type: string; + ownerId: string; +} + +function getSignatureKey( + key: string, + dateStamp: string, + regionName: string, + serviceName: string, +) { + const kDate = CryptoJS.HmacSHA256(dateStamp, 'AWS4' + key); + const kRegion = CryptoJS.HmacSHA256(regionName, kDate); + const kService = CryptoJS.HmacSHA256(serviceName, kRegion); + const kSigning = CryptoJS.HmacSHA256('aws4_request', kService); + return kSigning; +} + +function generateAWSHeaders( + url: string, + method: string, + service: string, + region: string, + accessKey: string, + secretKey: string, + sessionToken: string, + algorithm: string, + x_amz_token: string, +) { + const amzDate = new Date() + .toISOString() + .replace(/[:-]/g, '') + .replace(/\.\d{3}/, ''); + const dateStamp = amzDate.slice(0, 8); + const canonicalUri = new URL(url).pathname; + const canonicalQueryString = ''; + const payloadHash = CryptoJS.SHA256('').toString(CryptoJS.enc.Hex); + const canonicalHeaders = + `host:${new URL(url).hostname}\nx-amz-date:${amzDate}\n` + + (sessionToken ? `x-amz-security-token:${sessionToken}\n` : ''); + const signedHeaders = + 'host;x-amz-date' + (sessionToken ? `;${x_amz_token}` : ''); + const canonicalRequest = `${method}\n${canonicalUri}\n${canonicalQueryString}\n${canonicalHeaders}\n${signedHeaders}\n${payloadHash}`; + + const credentialScope = `${dateStamp}/${region}/${service}/aws4_request`; + const stringToSign = `${algorithm}\n${amzDate}\n${credentialScope}\n${CryptoJS.SHA256(canonicalRequest).toString(CryptoJS.enc.Hex)}`; + + const signingKey = getSignatureKey(secretKey, dateStamp, region, service); + const signature = CryptoJS.HmacSHA256(stringToSign, signingKey).toString( + CryptoJS.enc.Hex, + ); + const authorizationHeader = `${algorithm} Credential=${accessKey}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`; + + return { + host: new URL(url).hostname, + 'x-amz-date': amzDate, + 'x-amz-content-sha256': payloadHash, + Authorization: authorizationHeader, + ...(sessionToken && { 'x-amz-security-token': sessionToken }), + }; +} + +export async function fetchFromS3( + url: string, + method: string, + service: string, + region: string, + accessKey: string, + secretKey: string, + sessionToken: string, + algorithm: string, + x_amz_token: string, +) { + const headers = generateAWSHeaders( + url, + method, + service, + region, + accessKey, + secretKey, + sessionToken, + algorithm, + x_amz_token, + ); + + const response = await fetch(url, { + method, + headers, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch from S3: ${response.statusText}`); + } + + const data = await response.text(); + return data; +} + +export async function listBucketContents( + endpoint: string, + bucket: string, + directory: string, + method: string, + service: string, + region: string, + accessKey: string, + secretKey: string, + sessionToken: string, + algorithm: string, + x_amz_token: string, +) { + const listData = await fetchFromS3( + `${endpoint}/${bucket}`, + method, + service, + region, + accessKey, + secretKey, + sessionToken, + algorithm, + x_amz_token, + ); + + const parsedListData = await parseStringPromise(listData); + const contents = parsedListData.ListBucketResult.Contents; + + const fileNames = contents + .filter((item) => item.Key[0].startsWith(`${directory}/`)) + .map((item) => item.Key[0]); + + return fileNames.sort().reverse()[0]; // Get the latest file name +} + +export async function parseCSVData(fileData: string, ownerId: string) { + return new Promise((resolve, reject) => { + const itemsToInsert: CSVType[] = []; + + Papa.parse(fileData, { + header: true, + complete: (results) => { + try { + for (const [index, item] of results.data.entries()) { + if ( + index === results.data.length - 1 && + Object.values(item).every((value) => value === '') + ) { + continue; + } + + itemsToInsert.push({ + name: item.name, + claimed_weight: item.claimed_weight || 0, + quantity: item.quantity || 1, + claimed_weight_unit: item.claimed_weight_unit || 'g', + type: 'Essentials', + ownerId, + }); + } + resolve(itemsToInsert); + } catch (error) { + reject(error); + } + }, + error: (error) => reject(error), + }); + }); +} diff --git a/server/src/services/item/item.service.ts b/server/src/services/item/item.service.ts index 31924e039..fb24b7bc3 100644 --- a/server/src/services/item/item.service.ts +++ b/server/src/services/item/item.service.ts @@ -10,3 +10,4 @@ export * from './getItemsGloballyService'; export * from './searchItemsByNameService'; export * from './getItemsService'; export * from './bulkAddGlobalItemService'; +export * from './importFromBucketService';