From 0c8ec1e68078514c86b3de23e60370e724673dd0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 07:53:43 +0000 Subject: [PATCH 1/3] chore: bump up prisma monorepo to v6 --- packages/backend/server/package.json | 6 +- yarn.lock | 88 ++++++++++++++-------------- 2 files changed, 47 insertions(+), 47 deletions(-) diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 5f370197a4355..f154f51b07e8e 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -54,8 +54,8 @@ "@opentelemetry/sdk-node": "^0.55.0", "@opentelemetry/sdk-trace-node": "^1.25.0", "@opentelemetry/semantic-conventions": "^1.25.0", - "@prisma/client": "^5.15.0", - "@prisma/instrumentation": "^5.15.0", + "@prisma/client": "^6.0.0", + "@prisma/instrumentation": "^6.0.0", "@socket.io/redis-adapter": "^8.3.0", "cookie-parser": "^1.4.6", "dotenv": "^16.4.5", @@ -79,7 +79,7 @@ "on-headers": "^1.0.2", "openai": "^4.33.0", "piscina": "^4.5.1", - "prisma": "^5.12.1", + "prisma": "^6.0.0", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", "ses": "^1.4.1", diff --git a/yarn.lock b/yarn.lock index 579f14b18d3a3..cb00032af7d84 100644 --- a/yarn.lock +++ b/yarn.lock @@ -807,8 +807,8 @@ __metadata: "@opentelemetry/sdk-node": "npm:^0.55.0" "@opentelemetry/sdk-trace-node": "npm:^1.25.0" "@opentelemetry/semantic-conventions": "npm:^1.25.0" - "@prisma/client": "npm:^5.15.0" - "@prisma/instrumentation": "npm:^5.15.0" + "@prisma/client": "npm:^6.0.0" + "@prisma/instrumentation": "npm:^6.0.0" "@socket.io/redis-adapter": "npm:^8.3.0" "@types/cookie-parser": "npm:^1.4.7" "@types/express": "npm:^4.17.21" @@ -847,7 +847,7 @@ __metadata: on-headers: "npm:^1.0.2" openai: "npm:^4.33.0" piscina: "npm:^4.5.1" - prisma: "npm:^5.12.1" + prisma: "npm:^6.0.0" reflect-metadata: "npm:^0.2.2" rxjs: "npm:^7.8.1" ses: "npm:^1.4.1" @@ -9592,61 +9592,61 @@ __metadata: languageName: node linkType: hard -"@prisma/client@npm:^5.15.0": - version: 5.22.0 - resolution: "@prisma/client@npm:5.22.0" +"@prisma/client@npm:^6.0.0": + version: 6.0.1 + resolution: "@prisma/client@npm:6.0.1" peerDependencies: prisma: "*" peerDependenciesMeta: prisma: optional: true - checksum: 10/991d7410ded2d6c824303ac222ae94df4f1bf42240dda97c710cfee3a3bc59fef65c73e1ee55177e38d12f4d7dfc0a047748f98b4e3026dddd65f462684cd2d1 + checksum: 10/de537c1ef3de528b2dfa93d661d16ff47a14e688dc0e26bc97d0cf3775e7049325cd36120308e0a85bea5188cbc4be9966e9ca9df3c4508e689830082e2f3a64 languageName: node linkType: hard -"@prisma/debug@npm:5.22.0": - version: 5.22.0 - resolution: "@prisma/debug@npm:5.22.0" - checksum: 10/e4c425fde57f83c1a3df44d3f9088684a3e1d764022d2378f981209e57b7f421bc773d7f4fc23daa3936aa3e3772fa99fef81d2f2ec9673269f06efe822e3b53 +"@prisma/debug@npm:6.0.1": + version: 6.0.1 + resolution: "@prisma/debug@npm:6.0.1" + checksum: 10/f0c232faea07c73e85ddd0269db4b345fb3826bdf5d7e0c00702121037fde715d1b66f2d9cee18201b20d4284295c3555d47254622bdde604ec6dff5da4dad64 languageName: node linkType: hard -"@prisma/engines-version@npm:5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2": - version: 5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2 - resolution: "@prisma/engines-version@npm:5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2" - checksum: 10/e9e3563d75482591ca482dfc26afaf8a727796f48bfcc9b550652ca9c66176f5841944d00f8a4cb60a7fd58b117d8e28b1c6b84fc7316123738f7290c91c291d +"@prisma/engines-version@npm:5.23.0-27.5dbef10bdbfb579e07d35cc85fb1518d357cb99e": + version: 5.23.0-27.5dbef10bdbfb579e07d35cc85fb1518d357cb99e + resolution: "@prisma/engines-version@npm:5.23.0-27.5dbef10bdbfb579e07d35cc85fb1518d357cb99e" + checksum: 10/f38106e183943014b8a7c5c7b748298ea598dbb10fba9d7cd14234daacbebdb64dda3a87d4c148a3dcca5028b3721dfdd4e506910d182771fa3c211ea279d9f4 languageName: node linkType: hard -"@prisma/engines@npm:5.22.0": - version: 5.22.0 - resolution: "@prisma/engines@npm:5.22.0" +"@prisma/engines@npm:6.0.1": + version: 6.0.1 + resolution: "@prisma/engines@npm:6.0.1" dependencies: - "@prisma/debug": "npm:5.22.0" - "@prisma/engines-version": "npm:5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2" - "@prisma/fetch-engine": "npm:5.22.0" - "@prisma/get-platform": "npm:5.22.0" - checksum: 10/887381d8be0acc713159ef70cdc5c4b2ca5af6f3a5d2f6ae73b63c6cf31b74e5c41a9c2dbccb02b1a49bbda19a3d0a75cd1bcc86d76e98991c3a3978e61b622b + "@prisma/debug": "npm:6.0.1" + "@prisma/engines-version": "npm:5.23.0-27.5dbef10bdbfb579e07d35cc85fb1518d357cb99e" + "@prisma/fetch-engine": "npm:6.0.1" + "@prisma/get-platform": "npm:6.0.1" + checksum: 10/f70bd915df0996feba5e564dfc237c69a704ff9275773c32823f70277a130f62b17d9c0927b3761f8430555a2c9843a24cbc6351d107b7fbc2854e3f0ae03f9f languageName: node linkType: hard -"@prisma/fetch-engine@npm:5.22.0": - version: 5.22.0 - resolution: "@prisma/fetch-engine@npm:5.22.0" +"@prisma/fetch-engine@npm:6.0.1": + version: 6.0.1 + resolution: "@prisma/fetch-engine@npm:6.0.1" dependencies: - "@prisma/debug": "npm:5.22.0" - "@prisma/engines-version": "npm:5.22.0-44.605197351a3c8bdd595af2d2a9bc3025bca48ea2" - "@prisma/get-platform": "npm:5.22.0" - checksum: 10/512ce722e582a72429c90d438a63c730b230faa1235300215ab6cd1c5a0dfb2aa63704a66ee72c57222d9c094477ea198b1c154b9779dcab78ebf36e09f79161 + "@prisma/debug": "npm:6.0.1" + "@prisma/engines-version": "npm:5.23.0-27.5dbef10bdbfb579e07d35cc85fb1518d357cb99e" + "@prisma/get-platform": "npm:6.0.1" + checksum: 10/6c177443c828e1bab077caeac48c5b51601d48c58dc5ef3fb7825d96352708d6ad00827a85e5eab026ac30b1180e1f3e39c98e93f414b57e34235a0712e11ff0 languageName: node linkType: hard -"@prisma/get-platform@npm:5.22.0": - version: 5.22.0 - resolution: "@prisma/get-platform@npm:5.22.0" +"@prisma/get-platform@npm:6.0.1": + version: 6.0.1 + resolution: "@prisma/get-platform@npm:6.0.1" dependencies: - "@prisma/debug": "npm:5.22.0" - checksum: 10/15db9f38933a59a1b0f3a1d6284a50261803677516d6eedec2f7caaa1767c1c6e94e6465aca9239766b1cc363002476da21f7eeab55cc5329937d6d6d57a5f67 + "@prisma/debug": "npm:6.0.1" + checksum: 10/ab684fb05998374dc689b28775329742f4a91435b091f337bb6eff8ba9eeb0236fdf8365ce19b29948232adedbad0ba6cf1554d7f809009b5c0f3ea421e3e8cb languageName: node linkType: hard @@ -9661,14 +9661,14 @@ __metadata: languageName: node linkType: hard -"@prisma/instrumentation@npm:^5.15.0": - version: 5.22.0 - resolution: "@prisma/instrumentation@npm:5.22.0" +"@prisma/instrumentation@npm:^6.0.0": + version: 6.0.1 + resolution: "@prisma/instrumentation@npm:6.0.1" dependencies: "@opentelemetry/api": "npm:^1.8" "@opentelemetry/instrumentation": "npm:^0.49 || ^0.50 || ^0.51 || ^0.52.0 || ^0.53.0" "@opentelemetry/sdk-trace-base": "npm:^1.22" - checksum: 10/f48fc6b56e17538013033b5cab651d5d8df8bd0a0695ac3bc0d0cc6619a262a280ffe55aab0acade146928c6c2dfdf5532155a792818c5aea15efced67cc19c1 + checksum: 10/a2dabfd32c83848e2791b6c0b6b4ac712f543a3a66b60f282b1e51bd804b17130befa1f28c37e7b3e1cd1b25ce3b36e4df6bb543eadf331586a9d8d1bd1746fe languageName: node linkType: hard @@ -28029,18 +28029,18 @@ __metadata: languageName: node linkType: hard -"prisma@npm:^5.12.1": - version: 5.22.0 - resolution: "prisma@npm:5.22.0" +"prisma@npm:^6.0.0": + version: 6.0.1 + resolution: "prisma@npm:6.0.1" dependencies: - "@prisma/engines": "npm:5.22.0" + "@prisma/engines": "npm:6.0.1" fsevents: "npm:2.3.3" dependenciesMeta: fsevents: optional: true bin: prisma: build/index.js - checksum: 10/7d7a47491b6b5fb6fe23358af4ab36c1b85e837bf439c6994cfb20e98627dd15e7bc9e1be40f9620170fc587e6d0b5c6b871f44e5a380b15b7b684eb89e21037 + checksum: 10/276ef8120c75813b7b846c7eee1cf70dc5d0c3ecf5b16b3568cb69c728d401d620f822dfd7b143706493e05712d8512b57fdc3619e7077f85771477d8698284c languageName: node linkType: hard From ebc683fa62bbc83c336ed392a5f1633d46a5e282 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:06:52 +0000 Subject: [PATCH 2/3] refactor: update Buffer usages to Uint8Array Co-Authored-By: LongYinan --- .../backend/server/src/core/doc/options.ts | 26 +++++++++---- .../server/src/core/storage/wrappers/blob.ts | 2 +- .../backend/server/src/core/sync/gateway.ts | 21 ++++++---- .../src/core/workspaces/resolvers/blob.ts | 4 +- .../core/workspaces/resolvers/workspace.ts | 6 +-- .../storage/providers/provider.ts | 2 +- .../fundamentals/storage/providers/utils.ts | 39 +++++++++++++------ packages/frontend/native/index.d.ts | 1 - packages/frontend/native/index.js | 5 ++- 9 files changed, 71 insertions(+), 35 deletions(-) diff --git a/packages/backend/server/src/core/doc/options.ts b/packages/backend/server/src/core/doc/options.ts index 46523af9a4f3e..ea1d45151bcd9 100644 --- a/packages/backend/server/src/core/doc/options.ts +++ b/packages/backend/server/src/core/doc/options.ts @@ -12,8 +12,12 @@ import { PermissionService } from '../permission'; import { QuotaService } from '../quota'; import { DocStorageOptions as IDocStorageOptions } from './storage'; -function compare(yBinary: Buffer, jwstBinary: Buffer, strict = false): boolean { - if (yBinary.equals(jwstBinary)) { +function compare( + yBinary: Uint8Array, + jwstBinary: Uint8Array, + strict = false +): boolean { + if (Buffer.from(yBinary).equals(Buffer.from(jwstBinary))) { return true; } @@ -24,7 +28,7 @@ function compare(yBinary: Buffer, jwstBinary: Buffer, strict = false): boolean { const doc = new Y.Doc(); Y.applyUpdate(doc, jwstBinary); - const yBinary2 = Buffer.from(Y.encodeStateAsUpdate(doc)); + const yBinary2 = new Uint8Array(Y.encodeStateAsUpdate(doc)); return compare(yBinary, yBinary2, true); } @@ -41,7 +45,7 @@ export class DocStorageOptions implements IDocStorageOptions { mergeUpdates = async (updates: Uint8Array[]) => { const doc = await this.recoverDoc(updates); - const yjsResult = Buffer.from(Y.encodeStateAsUpdate(doc)); + const yjsResult = new Uint8Array(Y.encodeStateAsUpdate(doc)); const useYocto = await this.config.runtime.fetch( 'doc/experimentalMergeWithYOcto' @@ -50,16 +54,22 @@ export class DocStorageOptions implements IDocStorageOptions { if (useYocto) { metrics.jwst.counter('codec_merge_counter').add(1); let log = false; - let yoctoResult: Buffer | null = null; + let yoctoResult: Uint8Array | null = null; try { - yoctoResult = yotcoMergeUpdates(updates.map(Buffer.from)); + yoctoResult = new Uint8Array( + yotcoMergeUpdates(updates.map(u => Buffer.from(u))) + ); if (!compare(yjsResult, yoctoResult)) { metrics.jwst.counter('codec_not_match').add(1); this.logger.warn(`yocto codec result doesn't match yjs codec result`); log = true; if (this.config.node.dev) { - this.logger.warn(`Expected:\n ${yjsResult.toString('hex')}`); - this.logger.warn(`Result:\n ${yoctoResult.toString('hex')}`); + this.logger.warn( + `Expected:\n ${Buffer.from(yjsResult).toString('hex')}` + ); + this.logger.warn( + `Result:\n ${Buffer.from(yoctoResult).toString('hex')}` + ); } } } catch (e) { diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.ts index edc4b685ab65f..1df275c5494a2 100644 --- a/packages/backend/server/src/core/storage/wrappers/blob.ts +++ b/packages/backend/server/src/core/storage/wrappers/blob.ts @@ -28,7 +28,7 @@ export class WorkspaceBlobStorage { this.provider = this.storageFactory.create(this.config.storages.blob); } - async put(workspaceId: string, key: string, blob: Buffer) { + async put(workspaceId: string, key: string, blob: Uint8Array) { const meta: PutObjectMetadata = autoMetadata(blob); await this.provider.put(`${workspaceId}/${key}`, blob, meta); diff --git a/packages/backend/server/src/core/sync/gateway.ts b/packages/backend/server/src/core/sync/gateway.ts index d1c1c5ff7759a..d14118ec5d6f7 100644 --- a/packages/backend/server/src/core/sync/gateway.ts +++ b/packages/backend/server/src/core/sync/gateway.ts @@ -237,7 +237,9 @@ export class SpaceSyncGateway const doc = await adapter.diff( spaceId, docId, - stateVector ? Buffer.from(stateVector, 'base64') : undefined + stateVector + ? new Uint8Array(Buffer.from(stateVector, 'base64')) + : undefined ); if (!doc) { @@ -246,8 +248,8 @@ export class SpaceSyncGateway return { data: { - missing: Buffer.from(doc.missing).toString('base64'), - state: Buffer.from(doc.state).toString('base64'), + missing: Buffer.from(new Uint8Array(doc.missing)).toString('base64'), + state: Buffer.from(new Uint8Array(doc.state)).toString('base64'), timestamp: doc.timestamp, }, }; @@ -279,7 +281,7 @@ export class SpaceSyncGateway const timestamp = await adapter.push( spaceId, docId, - updates.map(update => Buffer.from(update, 'base64')), + updates.map(update => new Uint8Array(Buffer.from(update, 'base64'))), user.id ); @@ -323,7 +325,7 @@ export class SpaceSyncGateway const timestamp = await adapter.push( spaceId, docId, - [Buffer.from(update, 'base64')], + [new Uint8Array(Buffer.from(update, 'base64'))], user.id ); @@ -642,7 +644,12 @@ abstract class SyncSocketAdapter { permission?: Permission ): Promise; - push(spaceId: string, docId: string, updates: Buffer[], editorId: string) { + push( + spaceId: string, + docId: string, + updates: Uint8Array[], + editorId: string + ) { this.assertIn(spaceId); return this.storage.pushDocUpdates(spaceId, docId, updates, editorId); } @@ -675,7 +682,7 @@ class WorkspaceSyncAdapter extends SyncSocketAdapter { override push( spaceId: string, docId: string, - updates: Buffer[], + updates: Uint8Array[], editorId: string ) { const id = new DocID(docId, spaceId); diff --git a/packages/backend/server/src/core/workspaces/resolvers/blob.ts b/packages/backend/server/src/core/workspaces/resolvers/blob.ts index 448d23741c2cf..046a455dfbc23 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/blob.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/blob.ts @@ -112,7 +112,7 @@ export class WorkspaceBlobResolver { if (checkExceeded(0)) { throw new BlobQuotaExceeded(); } - const buffer = await new Promise((resolve, reject) => { + const buffer = await new Promise((resolve, reject) => { const stream = blob.createReadStream(); const chunks: Uint8Array[] = []; stream.on('data', chunk => { @@ -126,7 +126,7 @@ export class WorkspaceBlobResolver { }); stream.on('error', reject); stream.on('end', () => { - const buffer = Buffer.concat(chunks); + const buffer = new Uint8Array(Buffer.concat(chunks)); if (checkExceeded(buffer.length)) { reject(new BlobQuotaExceeded()); diff --git a/packages/backend/server/src/core/workspaces/resolvers/workspace.ts b/packages/backend/server/src/core/workspaces/resolvers/workspace.ts index b374e17cc0448..d7ab5697bd3f5 100644 --- a/packages/backend/server/src/core/workspaces/resolvers/workspace.ts +++ b/packages/backend/server/src/core/workspaces/resolvers/workspace.ts @@ -307,17 +307,17 @@ export class WorkspaceResolver { if (init) { // convert stream to buffer - const buffer = await new Promise(resolve => { + const buffer = await new Promise(resolve => { const stream = init.createReadStream(); const chunks: Uint8Array[] = []; stream.on('data', chunk => { chunks.push(chunk); }); stream.on('error', () => { - resolve(Buffer.from([])); + resolve(new Uint8Array()); }); stream.on('end', () => { - resolve(Buffer.concat(chunks)); + resolve(new Uint8Array(Buffer.concat(chunks))); }); }); diff --git a/packages/backend/server/src/fundamentals/storage/providers/provider.ts b/packages/backend/server/src/fundamentals/storage/providers/provider.ts index 5b2bad24b2473..d0db839ca4f71 100644 --- a/packages/backend/server/src/fundamentals/storage/providers/provider.ts +++ b/packages/backend/server/src/fundamentals/storage/providers/provider.ts @@ -24,7 +24,7 @@ export interface ListObjectsMetadata { contentLength: number; } -export type BlobInputType = Buffer | Readable | string; +export type BlobInputType = Buffer | Uint8Array | Readable | string; export type BlobOutputType = Readable; export interface StorageProvider { diff --git a/packages/backend/server/src/fundamentals/storage/providers/utils.ts b/packages/backend/server/src/fundamentals/storage/providers/utils.ts index 4c7e9323f2888..bd1b0a8cab795 100644 --- a/packages/backend/server/src/fundamentals/storage/providers/utils.ts +++ b/packages/backend/server/src/fundamentals/storage/providers/utils.ts @@ -7,15 +7,17 @@ import { getMime } from '../../../native'; import { BlobInputType, PutObjectMetadata } from './provider'; export async function toBuffer(input: BlobInputType): Promise { - return input instanceof Readable - ? await getStreamAsBuffer(input) - : input instanceof Buffer - ? input - : Buffer.from(input); + if (input instanceof Readable) { + return await getStreamAsBuffer(input); + } + if (input instanceof Buffer) { + return input; + } + return Buffer.from(input); } export function autoMetadata( - blob: Buffer, + blob: BlobInputType, raw: PutObjectMetadata = {} ): PutObjectMetadata { const metadata = { @@ -23,18 +25,33 @@ export function autoMetadata( }; if (!metadata.contentLength) { - metadata.contentLength = blob.byteLength; + metadata.contentLength = + blob instanceof Buffer || blob instanceof Uint8Array + ? blob.byteLength + : blob instanceof Readable + ? 0 + : Buffer.from(blob).byteLength; } try { - // checksum if (!metadata.checksumCRC32) { - metadata.checksumCRC32 = crc32(blob).toString(16); + metadata.checksumCRC32 = + blob instanceof Buffer + ? crc32(blob).toString(16) + : blob instanceof Uint8Array + ? crc32(Buffer.from(blob)).toString(16) + : blob instanceof Readable + ? undefined + : crc32(Buffer.from(blob)).toString(16); } - // mime type if (!metadata.contentType) { - metadata.contentType = getMime(blob); + metadata.contentType = + blob instanceof Buffer || blob instanceof Uint8Array + ? getMime(Buffer.from(blob)) + : blob instanceof Readable + ? 'application/octet-stream' + : getMime(Buffer.from(blob)); } } catch { // noop diff --git a/packages/frontend/native/index.d.ts b/packages/frontend/native/index.d.ts index c3611af9bfb2e..ed6c42562ec37 100644 --- a/packages/frontend/native/index.d.ts +++ b/packages/frontend/native/index.d.ts @@ -66,4 +66,3 @@ export declare enum ValidationResult { } export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise - diff --git a/packages/frontend/native/index.js b/packages/frontend/native/index.js index c6c15fe67f888..dac8b716b97d3 100644 --- a/packages/frontend/native/index.js +++ b/packages/frontend/native/index.js @@ -1,9 +1,12 @@ // prettier-ignore /* eslint-disable */ +// @ts-nocheck /* auto-generated by NAPI-RS */ -const { readFileSync } = require('fs') +const { createRequire } = require('node:module') +require = createRequire(__filename) +const { readFileSync } = require('node:fs') let nativeBinding = null const loadErrors = [] From 39852f4220217e75fab822e962bdde62a552b4d8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 14:17:24 +0000 Subject: [PATCH 3/3] chore: update native type definitions Co-Authored-By: LongYinan --- packages/backend/native/index.d.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index de691213ec201..c5d2e4e1d0cb1 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -19,4 +19,3 @@ export declare function mergeUpdatesInApplyWay(updates: Array): Buffer export declare function mintChallengeResponse(resource: string, bits?: number | undefined | null): Promise export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise -