diff --git a/packages/backend/server/src/__tests__/copilot.e2e.ts b/packages/backend/server/src/__tests__/copilot.e2e.ts index 48499e2b8d33f..7a6a3fc4e45a8 100644 --- a/packages/backend/server/src/__tests__/copilot.e2e.ts +++ b/packages/backend/server/src/__tests__/copilot.e2e.ts @@ -763,7 +763,11 @@ test('should be able to manage context', async t => { await t.notThrowsAsync(context, 'should create context with chat session'); const list = await listContext(app, token, workspaceId, sessionId); - t.deepEqual(list, [{ id: await context }], 'should list context'); + t.deepEqual( + list.map(f => ({ id: f.id })), + [{ id: await context }], + 'should list context' + ); } const fs = await import('node:fs'); diff --git a/packages/backend/server/src/__tests__/copilot.spec.ts b/packages/backend/server/src/__tests__/copilot.spec.ts index 47a82be0f3d76..86a8bc5b303fa 100644 --- a/packages/backend/server/src/__tests__/copilot.spec.ts +++ b/packages/backend/server/src/__tests__/copilot.spec.ts @@ -1294,8 +1294,8 @@ test('should be able to manage context', async t => { { const session = await context.create(chatSession); - const fileId = await session.add(file, randomUUID()); - const list = await session.listFiles(); + const fileId = await session.addFile(file, randomUUID()); + const list = session.listFiles(); t.deepEqual( list.map(f => f.chunk_size), [3], @@ -1309,10 +1309,10 @@ test('should be able to manage context', async t => { const docId = randomUUID(); await session.addDocRecord(randomUUID()); - const docs = await session.listDocs(); + const docs = session.listDocs(); t.deepEqual(docs, [docId], 'should list doc id'); - const result = await session.match('test', 2); + const result = await session.matchFileChunks('test', 2); t.is(result.length, 2, 'should match context'); t.is(result[0].fileId, fileId!, 'should match file id'); } diff --git a/packages/backend/server/src/__tests__/utils/copilot.ts b/packages/backend/server/src/__tests__/utils/copilot.ts index 591a77a611dde..4ef75730a9f19 100644 --- a/packages/backend/server/src/__tests__/utils/copilot.ts +++ b/packages/backend/server/src/__tests__/utils/copilot.ts @@ -306,7 +306,12 @@ export async function listContext( userToken: string, workspaceId: string, sessionId: string -): Promise<{ id: string }[]> { +): Promise< + { + id: string; + createdAt: number; + }[] +> { const res = await request(app.getHttpServer()) .post(gql) .auth(userToken, { type: 'bearer' }) @@ -318,6 +323,7 @@ export async function listContext( copilot(workspaceId: "${workspaceId}") { contexts(sessionId: "${sessionId}") { id + createdAt } } } @@ -407,6 +413,7 @@ export async function listContextFiles( blobId: string; chunk_size: number; status: string; + createdAt: number; }[] | undefined > { @@ -426,6 +433,7 @@ export async function listContextFiles( blobId chunk_size status + createdAt } } } diff --git a/packages/backend/server/src/plugins/copilot/context/resolver.ts b/packages/backend/server/src/plugins/copilot/context/resolver.ts index 1468c03b482d5..c4deb5f5a0b50 100644 --- a/packages/backend/server/src/plugins/copilot/context/resolver.ts +++ b/packages/backend/server/src/plugins/copilot/context/resolver.ts @@ -35,6 +35,7 @@ import { COPILOT_LOCKER, CopilotType } from '../resolver'; import { ChatSessionService } from '../session'; import { CopilotContextService } from './service'; import { + ContextDoc, type ContextFile, ContextFileStatus, DocChunkSimilarity, @@ -75,10 +76,22 @@ class RemoveContextFileInput { export class CopilotContextType { @Field(() => ID) id!: string; + + @Field(() => SafeIntResolver) + createdAt!: number; } registerEnumType(ContextFileStatus, { name: 'ContextFileStatus' }); +@ObjectType() +class CopilotContextDoc implements ContextDoc { + @Field(() => ID) + id!: string; + + @Field(() => SafeIntResolver) + createdAt!: number; +} + @ObjectType() class CopilotContextFile implements ContextFile { @Field(() => ID) @@ -95,6 +108,9 @@ class CopilotContextFile implements ContextFile { @Field(() => String) blobId!: string; + + @Field(() => SafeIntResolver) + createdAt!: number; } @ObjectType() @@ -252,17 +268,17 @@ export class CopilotContextResolver { return controller.signal; } - @ResolveField(() => [String], { + @ResolveField(() => [CopilotContextDoc], { description: 'list files in context', }) @CallMetric('ai', 'context_file_list') async docs( @Parent() context: CopilotContextType, @Args('contextId', { nullable: true }) contextId?: string - ): Promise { + ): Promise { const id = contextId || context.id; const session = await this.context.get(id); - return await session.listDocs(); + return session.listDocs(); } @Mutation(() => SafeIntResolver, { @@ -325,7 +341,7 @@ export class CopilotContextResolver { ): Promise { const id = contextId || context.id; const session = await this.context.get(id); - return await session.listFiles(); + return session.listFiles(); } @Mutation(() => String, { @@ -378,7 +394,7 @@ export class CopilotContextResolver { const session = await this.context.get(options.contextId); try { - return await session.remove(options.fileId); + return await session.removeFile(options.fileId); } catch (e: any) { throw new CopilotFailedToModifyContext({ contextId: options.contextId, @@ -406,7 +422,11 @@ export class CopilotContextResolver { const session = await this.context.get(contextId); try { - return await session.match(content, limit, this.getSignal(ctx.req)); + return await session.matchFileChunks( + content, + limit, + this.getSignal(ctx.req) + ); } catch (e: any) { throw new CopilotFailedToMatchContext({ contextId, @@ -433,7 +453,11 @@ export class CopilotContextResolver { await this.permissions.checkCloudWorkspace(session.workspaceId, user.id); try { - return await session.match(content, limit, this.getSignal(ctx.req)); + return await session.matchFileChunks( + content, + limit, + this.getSignal(ctx.req) + ); } catch (e: any) { throw new CopilotFailedToMatchContext({ contextId, diff --git a/packages/backend/server/src/plugins/copilot/context/service.ts b/packages/backend/server/src/plugins/copilot/context/service.ts index 669d63a3f2b36..d846181a75153 100644 --- a/packages/backend/server/src/plugins/copilot/context/service.ts +++ b/packages/backend/server/src/plugins/copilot/context/service.ts @@ -84,11 +84,14 @@ export class CopilotContextService { throw new CopilotInvalidContext({ contextId: id }); } - async list(sessionId: string): Promise<{ id: string }[]> { + async list(sessionId: string): Promise<{ id: string; createdAt: number }[]> { const contexts = await this.db.aiContext.findMany({ where: { sessionId }, - select: { id: true }, + select: { id: true, createdAt: true }, }); - return contexts; + return contexts.map(c => ({ + id: c.id, + createdAt: c.createdAt.getTime(), + })); } } diff --git a/packages/backend/server/src/plugins/copilot/context/session.ts b/packages/backend/server/src/plugins/copilot/context/session.ts index 1ca467ef619e9..37a575e5023ed 100644 --- a/packages/backend/server/src/plugins/copilot/context/session.ts +++ b/packages/backend/server/src/plugins/copilot/context/session.ts @@ -8,7 +8,9 @@ import { nanoid } from 'nanoid'; import { BlobQuotaExceeded, PrismaTransaction } from '../../../base'; import { OneMB } from '../../../core/quota/constant'; import { + ChunkSimilarity, ContextConfig, + ContextDoc, ContextFile, ContextFileStatus, DocChunkSimilarity, @@ -33,11 +35,11 @@ export class ContextSession implements AsyncDisposable { return this.config.workspaceId; } - async listDocs() { + listDocs(): ContextDoc[] { return [...this.config.docs]; } - async listFiles() { + listFiles() { return this.config.files.map(f => ({ ...f })); } @@ -65,6 +67,7 @@ export class ContextSession implements AsyncDisposable { blobId, chunk_size: embeddings.length, name, + createdAt: Date.now(), })); const values = this.processEmbeddings(fileId, embeddings); @@ -116,15 +119,15 @@ export class ContextSession implements AsyncDisposable { } async addDocRecord(docId: string) { - if (!this.config.docs.includes(docId)) { - this.config.docs.push(docId); + if (!this.config.docs.some(f => f.id === docId)) { + this.config.docs.push({ id: docId, createdAt: Date.now() }); await this.save(); } - return this.config.docs.length; + return this.config.docs; } async removeDocRecord(docId: string) { - const index = this.config.docs.indexOf(docId); + const index = this.config.docs.findIndex(f => f.id === docId); if (index >= 0) { this.config.docs.splice(index, 1); await this.save(); @@ -142,10 +145,10 @@ export class ContextSession implements AsyncDisposable { if (signal?.aborted) return; const buffer = await this.readStream(readable, 50 * OneMB); const file = new File([buffer], name); - return await this.add(file, blobId, signal); + return await this.addFile(file, blobId, signal); } - async add( + async addFile( file: File, blobId: string, signal?: AbortSignal @@ -157,7 +160,7 @@ export class ContextSession implements AsyncDisposable { return undefined; } - async remove(fileId: string) { + async removeFile(fileId: string) { return await this.db.$transaction(async tx => { const ret = await tx.aiContextEmbedding.deleteMany({ where: { contextId: this.contextId, fileId }, @@ -168,7 +171,7 @@ export class ContextSession implements AsyncDisposable { }); } - async match( + async matchFileChunks( content: string, topK: number = 5, signal?: AbortSignal @@ -185,11 +188,11 @@ export class ContextSession implements AsyncDisposable { `; } - async matchWorkspace( + async matchWorkspaceChunks( content: string, topK: number = 5, signal?: AbortSignal - ) { + ): Promise { const embedding = await this.client .getEmbeddings([content], signal) .then(r => r?.[0]?.embedding); diff --git a/packages/backend/server/src/plugins/copilot/context/types.ts b/packages/backend/server/src/plugins/copilot/context/types.ts index b165af97052e9..ecbe913efb57d 100644 --- a/packages/backend/server/src/plugins/copilot/context/types.ts +++ b/packages/backend/server/src/plugins/copilot/context/types.ts @@ -32,12 +32,19 @@ export const ContextConfigSchema = z.object({ ContextFileStatus.failed, ]), blobId: z.string(), + createdAt: z.number(), + }) + .array(), + docs: z + .object({ + id: z.string(), + createdAt: z.number(), }) .array(), - docs: z.string().array(), }); export type ContextConfig = z.infer; +export type ContextDoc = z.infer['docs'][number]; export type ContextFile = z.infer['files'][number]; export type ChunkSimilarity = { diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index 59e61e3bb2e0c..b8e9ffbd2d89b 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -78,17 +78,25 @@ type Copilot { } type CopilotContext { + createdAt: SafeInt! + """list files in context""" - docs(contextId: String): [String!]! + docs(contextId: String): [CopilotContextDoc!]! """list files in context""" files(contextId: String): [CopilotContextFile!]! id: ID! } +type CopilotContextDoc { + createdAt: SafeInt! + id: ID! +} + type CopilotContextFile { blobId: String! chunk_size: SafeInt! + createdAt: SafeInt! id: ID! name: String! status: ContextFileStatus! diff --git a/packages/frontend/graphql/src/graphql/copilot-context-file-list.gql b/packages/frontend/graphql/src/graphql/copilot-context-file-list.gql index 459fc6e0bba2f..a8981b5805069 100644 --- a/packages/frontend/graphql/src/graphql/copilot-context-file-list.gql +++ b/packages/frontend/graphql/src/graphql/copilot-context-file-list.gql @@ -6,13 +6,17 @@ query listContextFiles( currentUser { copilot(workspaceId: $workspaceId) { contexts(sessionId: $sessionId) { - docs(contextId: $contextId) + docs(contextId: $contextId) { + id + createdAt + } files(contextId: $contextId) { id name blobId chunk_size status + createdAt } } } diff --git a/packages/frontend/graphql/src/graphql/copilot-context-list.gql b/packages/frontend/graphql/src/graphql/copilot-context-list.gql index 049c4c3e7f67c..7effbe454935e 100644 --- a/packages/frontend/graphql/src/graphql/copilot-context-list.gql +++ b/packages/frontend/graphql/src/graphql/copilot-context-list.gql @@ -3,6 +3,7 @@ query listContext($workspaceId: String!, $sessionId: String!) { copilot(workspaceId: $workspaceId) { contexts(sessionId: $sessionId) { id + createdAt } } } diff --git a/packages/frontend/graphql/src/graphql/index.ts b/packages/frontend/graphql/src/graphql/index.ts index 8b771cbdfa9c7..e144719946d2d 100644 --- a/packages/frontend/graphql/src/graphql/index.ts +++ b/packages/frontend/graphql/src/graphql/index.ts @@ -198,13 +198,17 @@ query listContextFiles($workspaceId: String!, $sessionId: String!, $contextId: S currentUser { copilot(workspaceId: $workspaceId) { contexts(sessionId: $sessionId) { - docs(contextId: $contextId) + docs(contextId: $contextId) { + id + createdAt + } files(contextId: $contextId) { id name blobId chunk_size status + createdAt } } } @@ -250,6 +254,7 @@ query listContext($workspaceId: String!, $sessionId: String!) { copilot(workspaceId: $workspaceId) { contexts(sessionId: $sessionId) { id + createdAt } } } diff --git a/packages/frontend/graphql/src/schema.ts b/packages/frontend/graphql/src/schema.ts index c33df9067ecd4..ba92e1182cacf 100644 --- a/packages/frontend/graphql/src/schema.ts +++ b/packages/frontend/graphql/src/schema.ts @@ -127,8 +127,9 @@ export interface CopilotHistoriesArgs { export interface CopilotContext { __typename?: 'CopilotContext'; + createdAt: Scalars['SafeInt']['output']; /** list files in context */ - docs: Array; + docs: Array; /** list files in context */ files: Array; id: Scalars['ID']['output']; @@ -142,10 +143,17 @@ export interface CopilotContextFilesArgs { contextId?: InputMaybe; } +export interface CopilotContextDoc { + __typename?: 'CopilotContextDoc'; + createdAt: Scalars['SafeInt']['output']; + id: Scalars['ID']['output']; +} + export interface CopilotContextFile { __typename?: 'CopilotContextFile'; blobId: Scalars['String']['output']; chunk_size: Scalars['SafeInt']['output']; + createdAt: Scalars['SafeInt']['output']; id: Scalars['ID']['output']; name: Scalars['String']['output']; status: ContextFileStatus; @@ -368,6 +376,7 @@ export type ErrorDataUnion = | MemberNotFoundInSpaceDataType | MissingOauthQueryParameterDataType | NotInSpaceDataType + | QueryTooLongDataType | RuntimeConfigNotFoundDataType | SameSubscriptionRecurringDataType | SpaceAccessDeniedDataType @@ -445,6 +454,7 @@ export enum ErrorNames { OAUTH_STATE_EXPIRED = 'OAUTH_STATE_EXPIRED', PAGE_IS_NOT_PUBLIC = 'PAGE_IS_NOT_PUBLIC', PASSWORD_REQUIRED = 'PASSWORD_REQUIRED', + QUERY_TOO_LONG = 'QUERY_TOO_LONG', RUNTIME_CONFIG_NOT_FOUND = 'RUNTIME_CONFIG_NOT_FOUND', SAME_EMAIL_PROVIDED = 'SAME_EMAIL_PROVIDED', SAME_SUBSCRIPTION_RECURRING = 'SAME_SUBSCRIPTION_RECURRING', @@ -622,6 +632,15 @@ export interface InvoiceType { updatedAt: Scalars['DateTime']['output']; } +export interface License { + __typename?: 'License'; + expiredAt: Maybe; + installedAt: Scalars['DateTime']['output']; + quantity: Scalars['Int']['output']; + recurring: SubscriptionRecurring; + validatedAt: Scalars['DateTime']['output']; +} + export interface LimitedUserType { __typename?: 'LimitedUserType'; /** User email */ @@ -663,6 +682,7 @@ export interface MissingOauthQueryParameterDataType { export interface Mutation { __typename?: 'Mutation'; acceptInviteById: Scalars['Boolean']['output']; + activateLicense: License; /** add a doc to context */ addContextDoc: Scalars['SafeInt']['output']; /** add a file to context */ @@ -689,10 +709,12 @@ export interface Mutation { /** Create a stripe customer portal to manage payment methods */ createCustomerPortal: Scalars['String']['output']; createInviteLink: InviteLink; + createSelfhostWorkspaceCustomerPortal: Scalars['String']['output']; /** Create a new user */ createUser: UserType; /** Create a new workspace */ createWorkspace: WorkspaceType; + deactivateLicense: Scalars['Boolean']['output']; deleteAccount: DeleteAccount; deleteBlob: Scalars['Boolean']['output']; /** Delete a user account */ @@ -763,6 +785,11 @@ export interface MutationAcceptInviteByIdArgs { workspaceId: Scalars['String']['input']; } +export interface MutationActivateLicenseArgs { + license: Scalars['String']['input']; + workspaceId: Scalars['String']['input']; +} + export interface MutationAddContextDocArgs { options: AddContextDocInput; } @@ -834,6 +861,10 @@ export interface MutationCreateInviteLinkArgs { workspaceId: Scalars['String']['input']; } +export interface MutationCreateSelfhostWorkspaceCustomerPortalArgs { + workspaceId: Scalars['String']['input']; +} + export interface MutationCreateUserArgs { input: CreateUserInput; } @@ -842,6 +873,10 @@ export interface MutationCreateWorkspaceArgs { init?: InputMaybe; } +export interface MutationDeactivateLicenseArgs { + workspaceId: Scalars['String']['input']; +} + export interface MutationDeleteBlobArgs { hash?: InputMaybe; key?: InputMaybe; @@ -1187,6 +1222,11 @@ export interface QueryChatHistoriesInput { skip?: InputMaybe; } +export interface QueryTooLongDataType { + __typename?: 'QueryTooLongDataType'; + max: Scalars['Int']['output']; +} + export interface QuotaQueryType { __typename?: 'QuotaQueryType'; blobLimit: Scalars['SafeInt']['output']; @@ -1559,6 +1599,8 @@ export interface WorkspaceType { /** Get user invoice count */ invoiceCount: Scalars['Int']['output']; invoices: Array; + /** The selfhost license of the workspace */ + license: Maybe; /** member count of workspace */ memberCount: Scalars['Int']['output']; /** Members of workspace */ @@ -1600,6 +1642,7 @@ export interface WorkspaceTypeInvoicesArgs { } export interface WorkspaceTypeMembersArgs { + query?: InputMaybe; skip?: InputMaybe; take?: InputMaybe; } @@ -1792,7 +1835,11 @@ export type ListContextFilesQuery = { __typename?: 'Copilot'; contexts: Array<{ __typename?: 'CopilotContext'; - docs: Array; + docs: Array<{ + __typename?: 'CopilotContextDoc'; + id: string; + createdAt: number; + }>; files: Array<{ __typename?: 'CopilotContextFile'; id: string; @@ -1800,6 +1847,7 @@ export type ListContextFilesQuery = { blobId: string; chunk_size: number; status: ContextFileStatus; + createdAt: number; }>; }>; }; @@ -1843,7 +1891,11 @@ export type ListContextQuery = { __typename?: 'UserType'; copilot: { __typename?: 'Copilot'; - contexts: Array<{ __typename?: 'CopilotContext'; id: string }>; + contexts: Array<{ + __typename?: 'CopilotContext'; + id: string; + createdAt: number; + }>; }; } | null; };