From 1a4b114a393f56797296448443c7965bf8295a54 Mon Sep 17 00:00:00 2001 From: Maxime Bret Date: Mon, 9 Sep 2024 11:28:24 +0200 Subject: [PATCH 1/2] feat: added support for raw zip files --- packages/web/src/queries/queryClient.ts | 5 ++--- packages/web/src/reader/manifest/useManifest.ts | 4 ++++ packages/web/src/reader/streamer/archives.shared.ts | 6 +++++- packages/web/src/reader/streamer/webStreamer.ts | 4 +++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/web/src/queries/queryClient.ts b/packages/web/src/queries/queryClient.ts index c57bf470..c2298248 100644 --- a/packages/web/src/queries/queryClient.ts +++ b/packages/web/src/queries/queryClient.ts @@ -1,5 +1,4 @@ import { MutationCache, QueryCache, QueryClient } from "reactjrx" -import { Report } from "../debug/report.shared" import { isDebugEnabled } from "../debug/isDebugEnabled.shared" import { CancelError } from "../errors/errors.shared" @@ -12,14 +11,14 @@ export const queryClient = new QueryClient({ alert(String(error)) } - Report.error(error) + console.error(error) } }), queryCache: new QueryCache({ onError: (error) => { if (error instanceof CancelError) return - Report.error(error) + console.error(error) } }), defaultOptions: { diff --git a/packages/web/src/reader/manifest/useManifest.ts b/packages/web/src/reader/manifest/useManifest.ts index b1cc8213..41904517 100644 --- a/packages/web/src/reader/manifest/useManifest.ts +++ b/packages/web/src/reader/manifest/useManifest.ts @@ -30,6 +30,10 @@ export const useManifest = (bookId: string | undefined) => { baseUrl: getManifestBaseUrl(window.location.origin, bookId ?? "") }) + if (webStreamerResponse.status >= 400) { + throw webStreamerResponse + } + return { manifest: await webStreamerResponse.json(), isUsingWebStreamer: true diff --git a/packages/web/src/reader/streamer/archives.shared.ts b/packages/web/src/reader/streamer/archives.shared.ts index b6218f42..710735be 100644 --- a/packages/web/src/reader/streamer/archives.shared.ts +++ b/packages/web/src/reader/streamer/archives.shared.ts @@ -11,7 +11,11 @@ import { PromiseReturnType } from "../../types" import { Archive as LibARchive } from "libarchive.js" import { StreamerFileNotSupportedError } from "../../errors/errors.shared" -const jsZipCompatibleMimeTypes = [`application/epub+zip`, `application/x-cbz`] +const jsZipCompatibleMimeTypes = [ + `application/epub+zip`, + `application/x-cbz`, + `application/zip` +] const loadDataWithJsZip = async (data: Blob | File) => { try { diff --git a/packages/web/src/reader/streamer/webStreamer.ts b/packages/web/src/reader/streamer/webStreamer.ts index 2c4371a4..ab951520 100644 --- a/packages/web/src/reader/streamer/webStreamer.ts +++ b/packages/web/src/reader/streamer/webStreamer.ts @@ -22,7 +22,9 @@ export const webStreamer = new Streamer({ return await getArchiveForRarFile(file) } - return await getArchiveForZipFile(file) + const archive = await getArchiveForZipFile(file) + + return archive }, onError: onResourceError, onManifestSuccess From b51e4e2e734c13f4ddaf9711c91c9b55844da671 Mon Sep 17 00:00:00 2001 From: mbret Date: Sat, 28 Sep 2024 00:14:07 +0200 Subject: [PATCH 2/2] feat: improved error message --- package-lock.json | 4 +- package.json | 2 +- packages/api/package.json | 2 +- .../api/src/functions/corsProxy/handler.ts | 2 +- packages/api/src/functions/covers/handler.ts | 2 +- .../src/functions/refreshMetadata/handler.ts | 3 +- .../refreshMetadataCollection/handler.ts | 3 +- .../handler.ts | 2 +- .../refreshMetadataLongProcess/handler.ts | 2 +- .../src/functions/requestAccess/handler.ts | 2 +- packages/api/src/functions/signin/handler.ts | 1 - .../src/functions/syncDataSource/handler.ts | 3 +- .../syncDataSourceLongProcess/handler.ts | 2 +- .../api/src/functions/syncReports/handler.ts | 2 +- packages/api/src/libs/httpErrors.ts | 2 +- packages/api/src/libs/lambda.ts | 101 ------------------ packages/api/src/libs/middy/withMiddy.ts | 99 +++++++++++++++++ 17 files changed, 117 insertions(+), 117 deletions(-) create mode 100644 packages/api/src/libs/middy/withMiddy.ts diff --git a/package-lock.json b/package-lock.json index 78f5ba8b..24a2c9bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,7 +55,7 @@ "webpack-node-externals": "^3.0.0" }, "engines": { - "node": "18.x" + "node": "20.x" } }, "node_modules/@ampproject/remapping": { @@ -34519,7 +34519,7 @@ "webpack-node-externals": "^3.0.0" }, "engines": { - "node": "18.x" + "node": "20.x" } }, "packages/api/node_modules/fast-xml-parser": { diff --git a/package.json b/package.json index a9483527..de7f327e 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "author": "Maxime Bret ", "license": "MIT", "engines": { - "node": "18.x" + "node": "20.x" }, "workspaces": [ "packages/*" diff --git a/packages/api/package.json b/packages/api/package.json index 04b01a6e..48533322 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -15,7 +15,7 @@ "postinstall": "rm -rf node_modules/serverless" }, "engines": { - "node": "18.x" + "node": "20.x" }, "dependencies": { "node-unrar-js": "^2.0.2", diff --git a/packages/api/src/functions/corsProxy/handler.ts b/packages/api/src/functions/corsProxy/handler.ts index 88c794dc..f44007ad 100644 --- a/packages/api/src/functions/corsProxy/handler.ts +++ b/packages/api/src/functions/corsProxy/handler.ts @@ -1,8 +1,8 @@ import fetch, { AbortError } from "node-fetch" import { createHttpError } from "@libs/httpErrors" -import { withMiddy } from "@libs/lambda" import { APIGatewayProxyEvent } from "aws-lambda" import AbortController from "abort-controller" +import { withMiddy } from "@libs/middy/withMiddy" const lambda = async (event: APIGatewayProxyEvent) => { const params = event.queryStringParameters diff --git a/packages/api/src/functions/covers/handler.ts b/packages/api/src/functions/covers/handler.ts index 8a54712b..862cd619 100644 --- a/packages/api/src/functions/covers/handler.ts +++ b/packages/api/src/functions/covers/handler.ts @@ -1,10 +1,10 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { withMiddy } from "@libs/lambda" import { S3Client } from "@aws-sdk/client-s3" import sharp from "sharp" import { getCover } from "./getCover" import { getCoverPlaceholder } from "./getCoverPlaceholder" import createError from "http-errors" +import { withMiddy } from "@libs/middy/withMiddy" const s3 = new S3Client({ region: `us-east-1` diff --git a/packages/api/src/functions/refreshMetadata/handler.ts b/packages/api/src/functions/refreshMetadata/handler.ts index 8dcdf3fc..5fdb16ae 100644 --- a/packages/api/src/functions/refreshMetadata/handler.ts +++ b/packages/api/src/functions/refreshMetadata/handler.ts @@ -1,11 +1,12 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { getAwsLambda, withMiddy } from "@libs/lambda" +import { getAwsLambda } from "@libs/lambda" import { getNormalizedHeader } from "@libs/utils" import schema from "./schema" import { InvokeCommand } from "@aws-sdk/client-lambda" import { STAGE } from "src/constants" import { lock } from "@libs/supabase/lock" import { Logger } from "@libs/logger" +import { withMiddy } from "@libs/middy/withMiddy" const LOCK_MAX_DURATION_MN = 5 diff --git a/packages/api/src/functions/refreshMetadataCollection/handler.ts b/packages/api/src/functions/refreshMetadataCollection/handler.ts index f5cbc510..3ebd0c61 100644 --- a/packages/api/src/functions/refreshMetadataCollection/handler.ts +++ b/packages/api/src/functions/refreshMetadataCollection/handler.ts @@ -1,5 +1,5 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { getAwsLambda, withMiddy } from "@libs/lambda" +import { getAwsLambda } from "@libs/lambda" import { getNormalizedHeader } from "@libs/utils" import schema from "./schema" import { InvokeCommand } from "@aws-sdk/client-lambda" @@ -7,6 +7,7 @@ import { STAGE } from "src/constants" import { COLLECTION_METADATA_LOCK_MN } from "@oboku/shared" import { lock } from "@libs/supabase/lock" import { Logger } from "@libs/logger" +import { withMiddy } from "@libs/middy/withMiddy" const logger = Logger.child({ module: "handler" }) diff --git a/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts b/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts index 0ed02c68..12a9475f 100644 --- a/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts +++ b/packages/api/src/functions/refreshMetadataCollectionLongProcess/handler.ts @@ -1,5 +1,4 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { withMiddy } from "@libs/lambda" import { withToken } from "@libs/auth" import { configure as configureGoogleDataSource } from "@libs/plugins/google" import schema from "./schema" @@ -9,6 +8,7 @@ import { deleteLock } from "@libs/supabase/deleteLock" import { supabase } from "@libs/supabase/client" import { Logger } from "@libs/logger" import { refreshMetadata } from "./src/refreshMetadata" +import { withMiddy } from "@libs/middy/withMiddy" const lambda: ValidatedEventAPIGatewayProxyEvent = async ( event diff --git a/packages/api/src/functions/refreshMetadataLongProcess/handler.ts b/packages/api/src/functions/refreshMetadataLongProcess/handler.ts index c3ece21b..4cd9b51e 100644 --- a/packages/api/src/functions/refreshMetadataLongProcess/handler.ts +++ b/packages/api/src/functions/refreshMetadataLongProcess/handler.ts @@ -1,5 +1,4 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { withMiddy } from "@libs/lambda" import fs from "fs" import path from "path" import { OFFLINE, TMP_DIR } from "../../constants" @@ -13,6 +12,7 @@ import { deleteLock } from "@libs/supabase/deleteLock" import { supabase } from "@libs/supabase/client" import { Logger } from "@libs/logger" import { retrieveMetadataAndSaveCover } from "./src/retrieveMetadataAndSaveCover" +import { withMiddy } from "@libs/middy/withMiddy" const lambda: ValidatedEventAPIGatewayProxyEvent = async ( event diff --git a/packages/api/src/functions/requestAccess/handler.ts b/packages/api/src/functions/requestAccess/handler.ts index 014375fd..925dd683 100644 --- a/packages/api/src/functions/requestAccess/handler.ts +++ b/packages/api/src/functions/requestAccess/handler.ts @@ -1,10 +1,10 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" import { createHttpError } from "@libs/httpErrors" -import { withMiddy } from "@libs/lambda" import schema from "./schema" import nodemailer from "nodemailer" import { CONTACT_TO_ADDRESS } from "../../constants" import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm" +import { withMiddy } from "@libs/middy/withMiddy" const ssm = new SSMClient({ region: "us-east-1" }) diff --git a/packages/api/src/functions/signin/handler.ts b/packages/api/src/functions/signin/handler.ts index 3bd01d67..044aa9e1 100644 --- a/packages/api/src/functions/signin/handler.ts +++ b/packages/api/src/functions/signin/handler.ts @@ -5,7 +5,6 @@ * couchdb admin directly or probably by completely pruning db data. */ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { withMiddy } from "@libs/lambda" import schema from "./schema" import { initializeApp } from "firebase-admin/app" import { getAuth } from "firebase-admin/auth" diff --git a/packages/api/src/functions/syncDataSource/handler.ts b/packages/api/src/functions/syncDataSource/handler.ts index a647d692..dc0986c4 100644 --- a/packages/api/src/functions/syncDataSource/handler.ts +++ b/packages/api/src/functions/syncDataSource/handler.ts @@ -1,11 +1,12 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { getAwsLambda, withMiddy } from "@libs/lambda" +import { getAwsLambda } from "@libs/lambda" import { getNormalizedHeader } from "@libs/utils" import { STAGE } from "../../constants" import schema from "./schema" import { InvokeCommand } from "@aws-sdk/client-lambda" import { lock } from "@libs/supabase/lock" import { Logger } from "@libs/logger" +import { withMiddy } from "@libs/middy/withMiddy" const LOCK_MAX_DURATION_MN = 10 diff --git a/packages/api/src/functions/syncDataSourceLongProcess/handler.ts b/packages/api/src/functions/syncDataSourceLongProcess/handler.ts index bd4d287e..504a9b39 100644 --- a/packages/api/src/functions/syncDataSourceLongProcess/handler.ts +++ b/packages/api/src/functions/syncDataSourceLongProcess/handler.ts @@ -1,5 +1,4 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" -import { withMiddy } from "@libs/lambda" import { AWS_API_URI } from "../../constants" import { configure as configureGoogleDataSource } from "@libs/plugins/google" import { withToken } from "@libs/auth" @@ -12,6 +11,7 @@ import { deleteLock } from "@libs/supabase/deleteLock" import { supabase } from "@libs/supabase/client" import { pluginFacade } from "@libs/plugins/facade" import { Logger } from "@libs/logger" +import { withMiddy } from "@libs/middy/withMiddy" const logger = Logger.child({ module: "handler" }) diff --git a/packages/api/src/functions/syncReports/handler.ts b/packages/api/src/functions/syncReports/handler.ts index 3ec64050..193d3b0a 100644 --- a/packages/api/src/functions/syncReports/handler.ts +++ b/packages/api/src/functions/syncReports/handler.ts @@ -1,9 +1,9 @@ import { ValidatedEventAPIGatewayProxyEvent } from "@libs/api-gateway" import { withToken } from "@libs/auth" -import { withMiddy } from "@libs/lambda" import schema from "./schema" import { getParametersValue } from "@libs/ssm" import { supabase } from "@libs/supabase/client" +import { withMiddy } from "@libs/middy/withMiddy" const lambda: ValidatedEventAPIGatewayProxyEvent = async ( event diff --git a/packages/api/src/libs/httpErrors.ts b/packages/api/src/libs/httpErrors.ts index aa18f44f..46d4e238 100644 --- a/packages/api/src/libs/httpErrors.ts +++ b/packages/api/src/libs/httpErrors.ts @@ -1,7 +1,7 @@ import { ObokuErrorCode } from "@oboku/shared" import createError from "http-errors" -type ErrorEntry = { code: ObokuErrorCode } +type ErrorEntry = { code: ObokuErrorCode; message?: string } type ErrorEntries = ErrorEntry[] export const createHttpError = ( diff --git a/packages/api/src/libs/lambda.ts b/packages/api/src/libs/lambda.ts index 1967ba08..451d528f 100644 --- a/packages/api/src/libs/lambda.ts +++ b/packages/api/src/libs/lambda.ts @@ -1,107 +1,6 @@ -import middy from "@middy/core" -import { jsonSafeParse } from "@middy/util" -import middyJsonBodyParser from "@middy/http-json-body-parser" -import httpErrorHandler from "@middy/http-error-handler" -import httpHeaderNormalizer from "@middy/http-header-normalizer" -import cors from "@middy/http-cors" import { OFFLINE } from "../constants" -import { transpileSchema } from "@middy/validator/transpile" -import validator from "@middy/validator" import { LambdaClient } from "@aws-sdk/client-lambda" -export const withMiddy = ( - handler: any, - { - withCors = true, - withJsonBodyParser = true, - schema = {} - }: { - /** - * cors middleware only support REST / HTTP format. Some lambda are invoked from - * others lambda and therefore does not comply to the right format. These lambda - * can skip cors - */ - withCors?: boolean - withJsonBodyParser?: boolean - schema?: Parameters[0] - } = {} -) => { - const noop = { - before: () => {} - } - - return ( - middy(handler) - /** - * Some lambda are invoked from others lambda and therefore does not comply to the right format. These lambda. - * We make sure to have headers so the middy json does not fail - */ - .use({ - before: (request) => { - if (!request.event.headers) { - request.event.headers = {} - } - } - }) - .use(httpHeaderNormalizer()) - .use(withJsonBodyParser ? middyJsonBodyParser({}) : noop) - .use( - validator({ - eventSchema: transpileSchema(schema) - }) - ) - /** - * middy onError order changed and cors needs to be before to be executed after. - * Only for onError which is why it's duplicated below as well... - */ - .use({ - onError: withCors - ? cors({ - headers: `*` - }).onError - : () => { - // - } - }) - .use( - httpErrorHandler({ - // handle non http error with 500 and generic message - fallbackMessage: `An error occurred` - }) - ) - .use({ - onError: async (request) => { - if (request.error) { - console.error("error received", request.error) - } - // we enforce non exposure unless specified - if (request.error && (request.error as any)?.expose === undefined) { - // eslint-disable-next-line no-extra-semi - ;(request.error as any).expose = false - } - - // we force JSON response for any error that is a simple string - if ( - request.error && - typeof jsonSafeParse(request.error.message) === `string` - ) { - request.error.message = JSON.stringify({ - message: request.error.message - }) - } - } - }) - // @todo eventually protect the api and only allow a subset of origins - .use( - withCors - ? cors({ - headers: `*` - }) - : noop - ) - ) -} - export const getAwsLambda = () => new LambdaClient({ region: "us-east-1", diff --git a/packages/api/src/libs/middy/withMiddy.ts b/packages/api/src/libs/middy/withMiddy.ts new file mode 100644 index 00000000..2df74dd6 --- /dev/null +++ b/packages/api/src/libs/middy/withMiddy.ts @@ -0,0 +1,99 @@ +import middy from "@middy/core" +import { jsonSafeParse } from "@middy/util" +import middyJsonBodyParser from "@middy/http-json-body-parser" +import httpErrorHandler from "@middy/http-error-handler" +import httpHeaderNormalizer from "@middy/http-header-normalizer" +import cors from "@middy/http-cors" +import { transpileSchema } from "@middy/validator/transpile" +import validator from "@middy/validator" +import { createHttpError } from "@libs/httpErrors" +import { ObokuErrorCode } from "@oboku/shared" + +export const withMiddy = ( + handler: any, + { + withCors = true, + withJsonBodyParser = true, + schema = {} + }: { + /** + * cors middleware only support REST / HTTP format. Some lambda are invoked from + * others lambda and therefore does not comply to the right format. These lambda + * can skip cors + */ + withCors?: boolean + withJsonBodyParser?: boolean + schema?: Parameters[0] + } = {} +) => { + const noop = { + before: () => {} + } + + return ( + middy(handler) + /** + * Some lambda are invoked from others lambda and therefore does not comply to the right format. These lambda. + * We make sure to have headers so the middy json does not fail + */ + .use({ + before: (request) => { + if (!request.event.headers) { + request.event.headers = {} + } + } + }) + .use(httpHeaderNormalizer()) + .use(withJsonBodyParser ? middyJsonBodyParser({}) : noop) + .use( + validator({ + eventSchema: transpileSchema(schema) + }) + ) + /** + * middy onError order changed and cors needs to be before to be executed after. + * Only for onError which is why it's duplicated below as well... + */ + .use({ + onError: withCors + ? cors({ + headers: `*` + }).onError + : () => { + // + } + }) + .use( + httpErrorHandler({ + fallbackMessage: `An error occurred` + }) + ) + .use({ + onError: async (request) => { + if (request.error) { + console.error("error received", request.error) + } + + if (request.error) { + request.error = createHttpError(500, { + code: ObokuErrorCode.UNKNOWN, + message: request.error.message + }) + + // eslint-disable-next-line no-extra-semi + ;(request.error as any).expose = + (request.error as any)?.expose ?? + process.env.NODE_ENV === "development" + } + } + }) + // @todo eventually protect the api and only allow a subset of origins + .use( + withCors + ? cors({ + headers: `*` + }) + : noop + ) + ) +}