From 2f090be3af60cc4c231c1f9b213109681c372618 Mon Sep 17 00:00:00 2001 From: Tadjaur Date: Thu, 12 Dec 2024 20:51:54 +0100 Subject: [PATCH] Update and use existing item definition --- .../app/modules/item/hooks/useImportItem.ts | 2 + .../src/validations/itemRoutesValidator.ts | 26 ++- .../packTemplateRoutesValidator.ts | 8 +- .../src/controllers/item/importItemsGlobal.ts | 111 +++++++----- server/src/controllers/user/getUsers.ts | 4 +- .../src/services/item/addItemGlobalService.ts | 166 ++++++------------ server/src/services/item/addItemService.ts | 114 +----------- .../services/item/bulkAddGlobalItemService.ts | 147 +++++----------- .../packTemplate/addPackTemplateService.ts | 6 +- server/src/tests/routes/packTemplate.spec.ts | 4 + 10 files changed, 193 insertions(+), 395 deletions(-) diff --git a/packages/app/modules/item/hooks/useImportItem.ts b/packages/app/modules/item/hooks/useImportItem.ts index f673aa7d8..78b450a1c 100644 --- a/packages/app/modules/item/hooks/useImportItem.ts +++ b/packages/app/modules/item/hooks/useImportItem.ts @@ -16,6 +16,7 @@ export const useImportItem = () => { const handleImportNewItems = useCallback( (newItem, onSuccess) => { if (isConnected) { + console.log('isConnected'); return mutate(newItem, { onSuccess: () => { updateItems((prevState: State = {}) => { @@ -35,6 +36,7 @@ export const useImportItem = () => { }, }); } + console.log('not connected'); addOfflineRequest('addItemGlobal', newItem); diff --git a/packages/validations/src/validations/itemRoutesValidator.ts b/packages/validations/src/validations/itemRoutesValidator.ts index 0e0ba3c2a..95eb2bb6f 100644 --- a/packages/validations/src/validations/itemRoutesValidator.ts +++ b/packages/validations/src/validations/itemRoutesValidator.ts @@ -66,25 +66,19 @@ export const addItemGlobal = z.object({ name: z.string(), weight: z.number(), unit: z.string(), - type: z.string(), + type: z.enum(['Food', 'Water', 'Essentials']), ownerId: z.string(), + sku: z.string().optional(), + productUrl: z.string().optional(), + description: z.string().optional(), + productDetails: z + .record(z.string(), z.union([z.string(), z.number(), z.boolean()])) + .optional(), + seller: z.string().optional(), + image_urls: z.string().optional(), }); -export const addNewItemGlobal = z.object({ - name: z.string(), - weight: z.number(), - unit: z.string(), - category: z.enum(['Food', 'Water', 'Essentials']), - sku: z.string(), - productUrl: z.string(), - description: z.string(), - productDetails: z.record( - z.string(), - z.union([z.string(), z.number(), z.boolean()]), - ), - seller: z.string(), - imageUrls: z.string(), -}); +export type AddItemGlobalType = z.infer; export const importItemsGlobal = z.object({ content: z.string(), diff --git a/packages/validations/src/validations/packTemplateRoutesValidator.ts b/packages/validations/src/validations/packTemplateRoutesValidator.ts index 2458c9db5..de1c47541 100644 --- a/packages/validations/src/validations/packTemplateRoutesValidator.ts +++ b/packages/validations/src/validations/packTemplateRoutesValidator.ts @@ -1,5 +1,5 @@ import { z } from 'zod'; -import { addNewItemGlobal } from './itemRoutesValidator'; +import { addItemGlobal } from './itemRoutesValidator'; export const getPackTemplates = z.object({ filter: z @@ -23,7 +23,7 @@ export const getPackTemplate = z.union([ name: z.string(), id: z.string().optional(), }), -] as const); +]); export const createPackFromTemplate = z.object({ packTemplateId: z.string().min(1), @@ -37,10 +37,12 @@ export const addPackTemplate = z.object({ itemsOwnerId: z.string(), itemPackTemplates: z.array( z.object({ - item: addNewItemGlobal, + item: addItemGlobal.omit({ ownerId: true }), quantity: z.number(), }), ), }); +export type AddPackTemplateType = z.infer; + export const addPackTemplates = z.array(addPackTemplate); diff --git a/server/src/controllers/item/importItemsGlobal.ts b/server/src/controllers/item/importItemsGlobal.ts index fc15e1de9..50014a0e0 100644 --- a/server/src/controllers/item/importItemsGlobal.ts +++ b/server/src/controllers/item/importItemsGlobal.ts @@ -1,7 +1,7 @@ import { type Context } from 'hono'; import { addItemGlobalService, - addItemGlobalServiceBatch, + bulkAddItemsGlobalService, } from '../../services/item/item.service'; import { protectedProcedure } from '../../trpc'; import * as validator from '@packrat/validations'; @@ -72,6 +72,61 @@ export const importItemsGlobal = async (c: Context) => { } }; +/** + * Converts a list of raw CSV items into an iterable of validated items. + * @param {Array>} csvRawItems - The raw CSV items. + * @param {string} ownerId - The ID of the owner. + * @returns {Iterable} An iterable that yields the validated items. + */ +function* sanitizeItemsIterator( + csvRawItems: Array>, + ownerId: string, +): Generator { + for (let idx = 0; idx < csvRawItems.length; idx++) { + const item = csvRawItems[idx]; + + const productDetailsStr = `${item.techs}` + .replace(/'([^']*)'\s*:/g, '"$1":') // Replace single quotes keys with double quotes. + .replace(/:\s*'([^']*)'/g, ': "$1"') // Replace single quotes values with double quotes. + .replace(/\\x([0-9A-Fa-f]{2})/g, (match, hex) => { + // Replace hex escape sequences with UTF-8 characters + const codePoint = parseInt(hex, 16); + return String.fromCharCode(codePoint); + }); + + console.log(`${idx} / ${csvRawItems.length}`); + let parsedProductDetails: + | validator.AddItemGlobalType['productDetails'] + | null = null; + try { + parsedProductDetails = JSON.parse(productDetailsStr); + } catch (e) { + console.log( + `${productDetailsStr}\nFailed to parse product details for item ${item.Name}: ${e.message}`, + ); + throw e; + } + + const validatedItem: validator.AddItemGlobalType = { + name: String(item.Name), + weight: Number(item.Weight), + unit: String(item.Unit), + type: String(item.Category) as ItemCategoryEnum, + ownerId, + image_urls: item.image_urls && String(item.image_urls), + sku: item.sku && String(item.sku), + productUrl: item.product_url && String(item.product_url), + description: item.description && String(item.description), + seller: item.seller && String(item.seller), + }; + + if (parsedProductDetails) { + validatedItem.productDetails = parsedProductDetails; + } + + yield validatedItem; + } +} export function importItemsGlobalRoute() { const expectedHeaders = [ 'Name', @@ -114,49 +169,23 @@ export function importItemsGlobalRoute() { results.data.pop(); } - let idx = 0; - await addItemGlobalServiceBatch( - results.data, - true, + const errors: Error[] = []; + const createdItems = await bulkAddItemsGlobalService( + sanitizeItemsIterator(results.data, ownerId), opts.ctx.executionCtx, - (item) => { - const productDetailsStr = `${item.techs}` - .replace(/'([^']*)'\s*:/g, '"$1":') // Replace single quotes keys with double quotes. - .replace(/:\s*'([^']*)'/g, ': "$1"') // Replace single quotes values with double quotes. - .replace(/\\x([0-9A-Fa-f]{2})/g, (match, hex) => { - // Replace hex escape sequences with UTF-8 characters - const codePoint = parseInt(hex, 16); - return String.fromCharCode(codePoint); - }); - - idx++; - console.log(`${idx} / ${results.data.length}`); - try { - const parsedProductDetails = JSON.parse(productDetailsStr); - } catch (e) { - console.log( - `${productDetailsStr}\nFailed to parse product details for item ${item.Name}: ${e.message}`, - ); - throw e; - } - - return { - name: String(item.Name), - weight: Number(item.Weight), - unit: String(item.Unit), - type: String(item.Category) as ItemCategoryEnum, - ownerId, - executionCtx: opts.ctx.executionCtx, - image_urls: item.image_urls && String(item.image_urls), - sku: item.sku && String(item.sku), - productUrl: item.product_url && String(item.product_url), - description: item.description && String(item.description), - seller: item.seller && String(item.seller), - productDetails: JSON.parse(productDetailsStr), - }; + { + onItemCreationError: (error) => { + errors.push(error); + }, }, ); - return resolve('items'); + + return resolve({ + status: 'success', + items: createdItems, + errorsCount: errors.length, + errors, + }); } catch (error) { console.error(error); return reject(new Error(`Failed to add items: ${error.message}`)); diff --git a/server/src/controllers/user/getUsers.ts b/server/src/controllers/user/getUsers.ts index ece62d01e..6721b0655 100644 --- a/server/src/controllers/user/getUsers.ts +++ b/server/src/controllers/user/getUsers.ts @@ -1,4 +1,4 @@ -import { protectedProcedure, publicProcedure } from '../../trpc'; +import { protectedProcedure } from '../../trpc'; import { responseHandler } from '../../helpers/responseHandler'; import { User } from '../../drizzle/methods/User'; @@ -13,7 +13,7 @@ export const getUsers = async (c) => { }; export function getUsersRoute() { - return publicProcedure.query(async (opts) => { + return protectedProcedure.query(async (opts) => { const userClass = new User(); const users = await userClass.findMany(); return users; diff --git a/server/src/services/item/addItemGlobalService.ts b/server/src/services/item/addItemGlobalService.ts index 567c8390b..de2b7b323 100644 --- a/server/src/services/item/addItemGlobalService.ts +++ b/server/src/services/item/addItemGlobalService.ts @@ -1,4 +1,5 @@ import { type ExecutionContext } from 'hono'; +import * as validator from '@packrat/validations'; import { Item, ITEM_TABLE_NAME, @@ -9,145 +10,76 @@ import { ItemCategory } from '../../drizzle/methods/itemcategory'; import { ItemCategory as categories } from '../../utils/itemCategory'; import { VectorClient } from '../../vector/client'; import { convertWeight, SMALLEST_WEIGHT_UNIT } from 'src/utils/convertWeight'; -import { DbClient } from 'src/db/client'; -import { itemImage as itemImageTable } from '../../db/schema'; import { summarizeItem } from 'src/utils/item'; -import { addNewItemService, bulkAddNewItemsService } from './addItemService'; -// import { prisma } from '../../prisma'; -interface AddItemGlobalServiceParams { - /** The name of the item. */ - name: string; - /** The description of the item. */ - description?: string; - /** The weight of the item. */ - weight: number; - /** The unit of measurement for the item. */ - unit: string; - /** The category of the item. */ - type: (typeof categories)[number]; - /** The ID of the owner of the item. */ - ownerId: string; - /** The URLs of the images of the item. */ - image_urls?: string; - /** The SKU of the item. */ - sku?: string; - /** The URL of the product of the item. */ - productUrl?: string; - /** The product details of the item. */ - productDetails?: Record; - /** The seller of the item. */ - seller?: string; -} +type ItemWithCategory = Item & { category?: InsertItemCategory }; /** * Adds an item to the global service. + * @param {Object} item - The data for the new item. + * @param {ExecutionContext} executionCtx - Then vector database execution context * @return {Promise} The newly created item. */ export const addItemGlobalService = async ( - { - name, - weight, - unit, - type, - ownerId, - image_urls, - sku, - productUrl, - description, - productDetails, - seller, - }: AddItemGlobalServiceParams, - executionCtx: ExecutionContext, -) => { + item: validator.AddItemGlobalType, + executionCtx?: ExecutionContext, +): Promise => { let category: InsertItemCategory | null; - if (!categories.includes(type)) { - throw new Error(`Category must be one of: ${categories.join(', ')}`); + if (!categories.includes(item.type)) { + const error = new Error( + `[${item.sku}#${item.name}]: Category must be one of: ${categories.join(', ')}`, + ); + throw error; } + const itemClass = new ItemClass(); const itemCategoryClass = new ItemCategory(); - category = (await itemCategoryClass.findItemCategory({ name: type })) || null; + category = + (await itemCategoryClass.findItemCategory({ name: item.type })) || null; if (!category) { - category = await itemCategoryClass.create({ name: type }); + category = await itemCategoryClass.create({ name: item.type }); } - const newItem = await itemClass.create({ - name, - weight: convertWeight(Number(weight), unit as any, SMALLEST_WEIGHT_UNIT), - unit, + + const newItem = (await itemClass.create({ + name: item.name, + weight: convertWeight( + Number(item.weight), + item.unit as any, + SMALLEST_WEIGHT_UNIT, + ), + unit: item.unit, categoryId: category.id, global: true, - ownerId, - sku, - productUrl, - description, - productDetails, - seller, - }); + ownerId: item.ownerId, + sku: item.sku, + productUrl: item.productUrl, + description: item.description, + productDetails: item.productDetails, + seller: item.seller, + })) as ItemWithCategory; - if (image_urls) { - const urls = image_urls.split(','); + if (item.image_urls) { + const urls = item.image_urls.split(','); for (const url of urls) { - const newItemImage = { - itemId: newItem.id, - url, - }; - await DbClient.instance.insert(itemImageTable).values(newItemImage).run(); + await itemClass.insertImage(newItem.id, url); } } - executionCtx.waitUntil( - VectorClient.instance.syncRecord({ - id: newItem.id, - content: `product_name: ${name}, category: ${type}, description: ${description}, productDetails: ${JSON.stringify( - productDetails, - )}`, - namespace: ITEM_TABLE_NAME, - metadata: { - isPublic: newItem.global, - ownerId, - }, - }), - ); + newItem.category = category; - return newItem; -}; - -/** - * Adds list of items to the global service. - * @return {Promise} The newly created item. - */ -export const addItemGlobalServiceBatch = async ( - rawItems: T[], - continueOnError = false, - executionCtx: ExecutionContext, - transform: (rawItem: T) => AddItemGlobalServiceParams, -) => { - const errors: Error[] = []; - - const createdItemsInOrder: Array< - Awaited> - > = []; - - function* itemIterator() { - for (let idx = 0; idx < rawItems.length; idx++) { - const item = transform(rawItems[idx]); - yield { ...item, category: item.type, imageUrls: item.image_urls }; - } + if (executionCtx) { + executionCtx.waitUntil( + VectorClient.instance.syncRecord({ + id: newItem.id, + content: summarizeItem(newItem), + namespace: ITEM_TABLE_NAME, + metadata: { + isPublic: newItem.global, + ownerId: item.ownerId, + }, + }), + ); } - await bulkAddNewItemsService({ - items: itemIterator(), - executionCtx, - onItemCreationError: (error) => { - if (!continueOnError) { - throw error; - } - errors.push(error); - }, - }); - - return { - createdItemsInOrder, - errors, - }; + return newItem; }; diff --git a/server/src/services/item/addItemService.ts b/server/src/services/item/addItemService.ts index eb9344dfd..59f0652d2 100644 --- a/server/src/services/item/addItemService.ts +++ b/server/src/services/item/addItemService.ts @@ -1,17 +1,11 @@ import { type ExecutionContext } from 'hono'; -import * as validator from '@packrat/validations'; import { Item as ItemClass } from '../../drizzle/methods/Item'; import { ItemCategory } from '../../drizzle/methods/itemcategory'; import { ItemOwners } from '../../drizzle/methods/ItemOwners'; import { ItemCategory as categories } from '../../utils/itemCategory'; -import { - Item, - ITEM_TABLE_NAME, - type InsertItemCategory, -} from '../../db/schema'; +import { type InsertItemCategory } from '../../db/schema'; import { VectorClient } from '../../vector/client'; import { convertWeight, SMALLEST_WEIGHT_UNIT } from 'src/utils/convertWeight'; -import { summarizeItem } from 'src/utils/item'; /** * Generates a new item and adds it to a pack based on the given parameters. @@ -92,109 +86,3 @@ export const addItemService = async ( return item; }; - -/** - * Creates a new item. - * @param {Object} item - The data for the new item. - * @return {object} An object containing the newly created item - */ -export const addNewItemService = async ( - item: typeof validator.addNewItemGlobal._type, - ownerId: string, -): Promise => { - let category: InsertItemCategory | null; - if (!categories.includes(item.category)) { - const error = new Error( - `[${item.sku}#${item.name}]: Category must be one of: ${categories.join(', ')}`, - ); - throw error; - } - - const itemClass = new ItemClass(); - const itemCategoryClass = new ItemCategory(); - category = - (await itemCategoryClass.findItemCategory({ name: item.category })) || null; - if (!category) { - category = await itemCategoryClass.create({ name: item.category }); - } - - const newItem = await itemClass.create({ - name: item.name, - weight: convertWeight( - Number(item.weight), - item.unit as any, - SMALLEST_WEIGHT_UNIT, - ), - unit: item.unit, - categoryId: category.id, - global: true, - ownerId, - sku: item.sku, - productUrl: item.productUrl, - description: item.description, - productDetails: item.productDetails, - seller: item.seller, - }); - - if (item.imageUrls) { - const urls = item.imageUrls.split(','); - for (const url of urls) { - await itemClass.insertImage(newItem.id, url); - } - } - - return { ...newItem, category }; -}; - -/** - * Adds a list of items to the global inventory and indexes them in the vector database. - * @param {Object} bulkPram: the config parameter - * @param {Object} bulkPram.items - The list of items to add. - * @param {string} bulkPram.ownerId - The owner of all items - * @param {ExecutionContext} bulkParam.onItemCreated - A callback function to be called when an item is created. - * @param {Function} bulkParam.onItemCreated - A callback function to be called when an item is created. - */ -export const bulkAddNewItemsService = async (bulkParam: { - items: Iterable< - typeof validator.addNewItemGlobal._type & { ownerId: string } - >; - executionCtx: ExecutionContext; - onItemCreated?: ( - item: Awaited>, - bulkIndex: number, - ) => void; - onItemCreationError?: (error: Error, bulkIndex: number) => void; -}): Promise>>> => { - const { items, executionCtx, onItemCreated, onItemCreationError } = bulkParam; - const createdItems: Array>> = []; - const vectorData = []; - - let idx = -1; - for (const item of items) { - idx += 1; - try { - const createdItem = await addNewItemService(item, item.ownerId); - if (onItemCreated) { - onItemCreated(createdItem, idx); - } - createdItems.push(createdItem); - - vectorData.push({ - id: createdItem.id, - content: summarizeItem(createdItem), - namespace: ITEM_TABLE_NAME, - metadata: { - isPublic: createdItem.global, - ownerId: createdItem.ownerId, - }, - }); - } catch (error) { - if (onItemCreationError) { - onItemCreationError(error as Error, idx); - } - } - } - - executionCtx.waitUntil(VectorClient.instance.syncRecords(vectorData)); - return createdItems; -}; diff --git a/server/src/services/item/bulkAddGlobalItemService.ts b/server/src/services/item/bulkAddGlobalItemService.ts index d60ed3f40..c0aa1bb49 100644 --- a/server/src/services/item/bulkAddGlobalItemService.ts +++ b/server/src/services/item/bulkAddGlobalItemService.ts @@ -1,111 +1,60 @@ import { type ExecutionContext } from 'hono'; -import { type InsertItemCategory } from '../../db/schema'; -import { ItemCategory } from '../../drizzle/methods/itemcategory'; -import { DbClient } from 'src/db/client'; -import { - item as ItemTable, - itemImage as itemImageTable, -} from '../../db/schema'; -import { convertWeight, SMALLEST_WEIGHT_UNIT } from 'src/utils/convertWeight'; -import { eq } from 'drizzle-orm'; +import * as validator from '@packrat/validations'; +import { ITEM_TABLE_NAME } from '../../db/schema'; import { VectorClient } from 'src/vector/client'; - +import { addItemGlobalService } from './addItemGlobalService'; +import { summarizeItem } from 'src/utils/item'; + +/** + * Adds a list of items to the global inventory and indexes them in the vector database. + * @param {Object} items - The list of items to add. + * @param {string} executionCtx - The execution context. + * @param {ExecutionContext} callbacks.onItemCreated - A callback function to be called when an item is created. + * @param {Function} callbacks.onItemCreationError - A callback function to be called when an item creation fails. + * @return {Promise} A promise that resolves to an array of the created items. + */ export const bulkAddItemsGlobalService = async ( - items: Array<{ - name: string; - weight: number; - unit: string; - type: 'Food' | 'Water' | 'Essentials'; - ownerId: string; - image_urls?: string; - sku?: string; - productUrl?: string; - description?: string; - productDetails?: { - [key: string]: string | number | boolean | null; - }; - seller?: string; - }>, + items: Iterable, executionCtx: ExecutionContext, -) => { - const categories = ['Food', 'Water', 'Essentials']; - - const itemCategoryClass = new ItemCategory(); - const insertedItems = []; - - for (const itemData of items) { - const { name, weight, unit, type, ownerId, image_urls } = itemData; - if (!categories.includes(type)) { - throw new Error(`Category must be one of: ${categories.join(', ')}`); - } - - let category: InsertItemCategory | null; - category = - (await itemCategoryClass.findItemCategory({ name: type })) || null; - if (!category) { - category = await itemCategoryClass.create({ name: type }); - } - - // Check if item with the same name already exists - const existingItem = await DbClient.instance - .select() - .from(ItemTable) - .where(eq(ItemTable.name, name)) - .get(); - - if (existingItem) { - continue; - } - - const newItem = { - name, - weight: convertWeight(Number(weight), unit as any, SMALLEST_WEIGHT_UNIT), - unit, - categoryId: category.id, - global: true, - ownerId, - sku, - productUrl, - description, - productDetails, - seller, - }; - - const item = await DbClient.instance - .insert(ItemTable) - .values(newItem) - .returning() - .get(); + callbacks?: { + onItemCreated?: ( + item: Awaited>, + bulkIndex: number, + ) => void; + onItemCreationError?: (error: Error, bulkIndex: number) => void; + }, +): Promise>>> => { + const { onItemCreated, onItemCreationError } = callbacks; + const createdItems: Array>> = + []; + const vectorData = []; + + let idx = -1; + for (const item of items) { + idx += 1; + try { + const createdItem = await addItemGlobalService(item); + if (onItemCreated) { + onItemCreated(createdItem, idx); + } + createdItems.push(createdItem); - executionCtx.waitUntil( - VectorClient.instance.syncRecord({ - id: item.id, - content: name, - namespace: 'items', + vectorData.push({ + id: createdItem.id, + content: summarizeItem(createdItem), + namespace: ITEM_TABLE_NAME, metadata: { - isPublic: item.global, - ownerId, + isPublic: createdItem.global, + ownerId: createdItem.ownerId, }, - }), - ); - - if (image_urls) { - const urls = image_urls.split(','); - for (const url of urls) { - const newItemImage = { - itemId: item.id, - url, - }; - await DbClient.instance - .insert(itemImageTable) - .values(newItemImage) - .run(); + }); + } catch (error) { + if (onItemCreationError) { + onItemCreationError(error as Error, idx); } - console.log('Added image urls for item:', item.id); } - - insertedItems.push(item); } - return insertedItems; + executionCtx.waitUntil(VectorClient.instance.syncRecords(vectorData)); + return createdItems; }; diff --git a/server/src/services/packTemplate/addPackTemplateService.ts b/server/src/services/packTemplate/addPackTemplateService.ts index 49549c619..637a8083c 100644 --- a/server/src/services/packTemplate/addPackTemplateService.ts +++ b/server/src/services/packTemplate/addPackTemplateService.ts @@ -18,7 +18,7 @@ import { i } from 'vitest/dist/reporters-QGe8gs4b.js'; * @return {Object} An object containing the created pack. */ export const addPackTemplateService = async ( - packTemplateData: typeof validator.addPackTemplate._type, + packTemplateData: validator.AddPackTemplateType, executionCtx: ExecutionContext, ): Promise => { const { name, description, type } = packTemplateData; @@ -48,9 +48,7 @@ export const addPackTemplateService = async ( } } - await ItemService.bulkAddNewItemsService({ - items: itemIterator(), - executionCtx, + await ItemService.bulkAddItemsGlobalService(itemIterator(), executionCtx, { onItemCreationError: (error, idx) => { console.error(`Error creating item at ${idx}:`, error); }, diff --git a/server/src/tests/routes/packTemplate.spec.ts b/server/src/tests/routes/packTemplate.spec.ts index 5af0be490..d914ee981 100644 --- a/server/src/tests/routes/packTemplate.spec.ts +++ b/server/src/tests/routes/packTemplate.spec.ts @@ -123,6 +123,10 @@ describe('Pack template routes', () => { name: 'test', description: 'pack template description', type: 'pack', + itemPackTemplates: packTemplateItems.map((item) => ({ + itemId: item.id, + quantity: 1, + })), }); expect(packTemplate).toMatchObject([ { ...packTemplate, items: packTemplateItems },