From 047d8a95b9c9d1a608f6d55d5073236ed8d9bda3 Mon Sep 17 00:00:00 2001 From: sarthakpranesh Date: Sat, 9 Mar 2024 02:08:43 +0530 Subject: [PATCH 1/3] chore: adding mocked S3 bucket for local filestore Signed-off-by: sarthakpranesh --- .gitignore | 5 ++++- fileStore.ts | 47 +++++++++++++++++++++++++++++++++++++++++++++++ index.ts | 12 +++++++++--- 3 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 fileStore.ts diff --git a/.gitignore b/.gitignore index 1792221..d50e001 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,7 @@ node_modules # Output files -dist \ No newline at end of file +dist + +# local filestore +localFileStore \ No newline at end of file diff --git a/fileStore.ts b/fileStore.ts new file mode 100644 index 0000000..1c6e94a --- /dev/null +++ b/fileStore.ts @@ -0,0 +1,47 @@ + +import EventEmitter from 'node:events'; +import * as fs from 'node:fs'; + +export default (() => { + const base = './localFileStore/' + const metaPath = `${base}_meta.json` + if (fs.existsSync(base) === false) { + fs.mkdirSync(base) + } + if (fs.existsSync(metaPath) === false) { + fs.writeFileSync(metaPath, JSON.stringify({})) + } + return { + file: (filePath: string) => { + const fullPath = `${base}${filePath}` + return { + exists: () => { + return [fs.existsSync(fullPath)] + }, + download: async () => { + return [new Uint8Array(fs.readFileSync(fullPath))] + }, + createWriteStream: () => { + return fs.createWriteStream(fullPath) + }, + createReadStream: () => { + return fs.createReadStream(fullPath) + }, + save: (fileContent: string, meta?: undefined | any) => { + if (typeof meta === 'object') { + const rawMeta = fs.readFileSync(metaPath); + const _meta = JSON.parse(rawMeta.toString()) + _meta[fullPath] = meta.metadata + fs.writeFileSync(metaPath, JSON.stringify(_meta)) + } + fs.writeFileSync(fullPath, fileContent) + }, + getMetadata: () => { + const rawMeta = fs.readFileSync(metaPath); + const _meta = JSON.parse(rawMeta.toString()) + return [_meta[fullPath]] + } + } + } + } +})(); diff --git a/index.ts b/index.ts index 1d4b415..387efce 100644 --- a/index.ts +++ b/index.ts @@ -1,18 +1,24 @@ import { Database } from '@hocuspocus/extension-database'; import { Logger } from '@hocuspocus/extension-logger'; import { Server } from '@hocuspocus/server'; -import { Storage } from '@google-cloud/storage'; +import { Bucket, Storage } from '@google-cloud/storage'; import { utils } from './stream-helper' import cors from 'cors'; import express from 'express'; import expressWebsockets from 'express-ws'; import ShortUniqueId from 'short-unique-id'; +import fileStore from './fileStore'; const uid = new ShortUniqueId({ length: 20 }); const serverPort = parseInt(process.env.SERVER_PORT || '8080'); -const storageClient = new Storage(); -const storageBucket = storageClient.bucket(process.env.BUCKET_NAME || 'athene-diagram-files'); +let storageBucket: Bucket | typeof fileStore; +if (process.env.BUCKET_NAME) { + const storageClient = new Storage(); + storageBucket = storageClient.bucket(process.env.BUCKET_NAME); +} else { + storageBucket = fileStore +} const server = Server.configure({ extensions: [ From 6b001d0d540c584856dbc451dedd8d04daa727ae Mon Sep 17 00:00:00 2001 From: sarthakpranesh Date: Sun, 10 Mar 2024 12:34:17 +0530 Subject: [PATCH 2/3] chore: adding dotenv and reconfiguring conditions --- .env | 9 +++++++++ index.ts | 5 +++-- package-lock.json | 12 ++++++++++++ package.json | 1 + 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..1730186 --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +# port you want the server to use, defaults to 8080 +SERVER_PORT=8001 + +# you want to use gcp or local filesystem +# gcp -> google cloud storage, local -> local filesystem +STORAGE_TYPE=local + +# if storage type is gcp, then uncomment below and add the bucket name +# STORAGE_GCP_BUCKET_NAME=athene-diagram-files \ No newline at end of file diff --git a/index.ts b/index.ts index 387efce..08fc61b 100644 --- a/index.ts +++ b/index.ts @@ -1,3 +1,4 @@ +import 'dotenv/config' import { Database } from '@hocuspocus/extension-database'; import { Logger } from '@hocuspocus/extension-logger'; import { Server } from '@hocuspocus/server'; @@ -13,9 +14,9 @@ const uid = new ShortUniqueId({ length: 20 }); const serverPort = parseInt(process.env.SERVER_PORT || '8080'); let storageBucket: Bucket | typeof fileStore; -if (process.env.BUCKET_NAME) { +if (process.env.STORAGE_TYPE === "gcp") { const storageClient = new Storage(); - storageBucket = storageClient.bucket(process.env.BUCKET_NAME); + storageBucket = storageClient.bucket(process.env.STORAGE_GCP_BUCKET_NAME || 'athene-diagram-files'); } else { storageBucket = fileStore } diff --git a/package-lock.json b/package-lock.json index e225b85..0164244 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@hocuspocus/provider": "^2.7.1", "@hocuspocus/server": "^2.7.1", "cors": "^2.8.5", + "dotenv": "^16.4.5", "express": "^4.18.2", "express-ws": "^5.0.2", "short-unique-id": "^5.0.3", @@ -765,6 +766,17 @@ "node": ">=0.3.1" } }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/duplexify": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz", diff --git a/package.json b/package.json index a0ecb3a..6e1e380 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@hocuspocus/provider": "^2.7.1", "@hocuspocus/server": "^2.7.1", "cors": "^2.8.5", + "dotenv": "^16.4.5", "express": "^4.18.2", "express-ws": "^5.0.2", "short-unique-id": "^5.0.3", From 6d123f7d90913c1c853d81a2e85f916a1827641d Mon Sep 17 00:00:00 2001 From: sarthakpranesh Date: Sun, 10 Mar 2024 12:42:43 +0530 Subject: [PATCH 3/3] chore: reconfiguring for backward comp --- .env | 4 +++- index.ts | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 1730186..1662b88 100644 --- a/.env +++ b/.env @@ -3,7 +3,9 @@ SERVER_PORT=8001 # you want to use gcp or local filesystem # gcp -> google cloud storage, local -> local filesystem +# defaults to gcp STORAGE_TYPE=local -# if storage type is gcp, then uncomment below and add the bucket name +# if storage type is gcp/not set, then uncomment below and add the bucket name +# defaults to "athene-diagram-files" # STORAGE_GCP_BUCKET_NAME=athene-diagram-files \ No newline at end of file diff --git a/index.ts b/index.ts index 08fc61b..b3d2013 100644 --- a/index.ts +++ b/index.ts @@ -11,14 +11,14 @@ import ShortUniqueId from 'short-unique-id'; import fileStore from './fileStore'; const uid = new ShortUniqueId({ length: 20 }); - const serverPort = parseInt(process.env.SERVER_PORT || '8080'); + let storageBucket: Bucket | typeof fileStore; -if (process.env.STORAGE_TYPE === "gcp") { +if (process.env.STORAGE_TYPE === "local") { + storageBucket = fileStore +} else { const storageClient = new Storage(); storageBucket = storageClient.bucket(process.env.STORAGE_GCP_BUCKET_NAME || 'athene-diagram-files'); -} else { - storageBucket = fileStore } const server = Server.configure({