diff --git a/package.json b/package.json index 0a6acb2c..10f35558 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@webrecorder/wabac", - "version": "2.20.1", + "version": "2.20.2", "main": "index.js", "type": "module", "exports": { @@ -19,7 +19,7 @@ "@peculiar/asn1-schema": "^2.3.3", "@peculiar/x509": "^1.9.2", "@types/js-levenshtein": "^1.1.3", - "@webrecorder/wombat": "^3.8.3", + "@webrecorder/wombat": "^3.8.5", "acorn": "^8.10.0", "auto-js-ipfs": "^2.1.1", "base64-js": "^1.5.1", @@ -38,7 +38,7 @@ "path-parser": "^6.1.0", "process": "^0.11.10", "stream-browserify": "^3.0.0", - "warcio": "^2.3.1" + "warcio": "^2.4.0" }, "devDependencies": { "@swc-node/register": "^1.10.9", diff --git a/src/remotearchivedb.ts b/src/remotearchivedb.ts index da9bb28a..ca050c74 100644 --- a/src/remotearchivedb.ts +++ b/src/remotearchivedb.ts @@ -40,6 +40,26 @@ export abstract class OnDemandPayloadArchiveDB extends ArchiveDB { this.streamMap = new Map(); } + isSameUrl(remoteUrl: string, cdxUrl: string, method?: string | null) { + if (remoteUrl === cdxUrl) { + return true; + } + + const decodedRemoteUrl = decodeURIComponent(remoteUrl); + const decodedCdxUrl = decodeURIComponent(cdxUrl); + + if (decodedRemoteUrl === decodedCdxUrl) { + return true; + } + + // if non-GET, check if cdxUrl starts with requested url + if (method && decodedCdxUrl.startsWith(decodedRemoteUrl)) { + return true; + } + + return false; + } + abstract loadRecordFromSource( cdx: RemoteResourceEntry, ): LoadRecordFromSourceType; @@ -74,13 +94,7 @@ export abstract class OnDemandPayloadArchiveDB extends ArchiveDB { return null; } - if ( - remote.url !== cdx.url && - !( - cdx.method && - decodeURIComponent(cdx.url).startsWith(decodeURIComponent(remote.url)) - ) - ) { + if (!this.isSameUrl(remote.url, cdx.url, cdx.method)) { console.log(`Wrong url: expected ${cdx.url}, got ${remote.url}`); return null; } diff --git a/yarn.lock b/yarn.lock index 1d80cacf..c152e93e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -896,12 +896,12 @@ resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-1.5.2.tgz#ea584b637ff63c5a477f6f21604b5a205b72c9ec" integrity sha512-vgJ5OLWadI8aKjDlOH3rb+dYyPd2GTZuQC/Tihjct6F9GpXGZINo3Y/IVuZVTM1eDQB+/AOsjPUWH/WySDaXvw== -"@webrecorder/wombat@^3.8.3": - version "3.8.3" - resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.3.tgz#c5a077225d1a70def9fbbbfcd50fa4465d236546" - integrity sha512-dqgoxigB3OdX5JeB3yxJrUNwFwUBlYC+LmGrLEgGeP259MFzXQLD2pmfuqGt5ygWvIv56SrAMV4sUceux07X2A== +"@webrecorder/wombat@^3.8.5": + version "3.8.5" + resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.5.tgz#f62b18412c9e72d896c9604bbefe77c6fd9c43ea" + integrity sha512-yQDycwPtOmwNQUg/ro1xz4c5NWvgdszIllayYXV2WmyCJv27Gj1VJAXtBoEAVQffOl0Q14JVtMs+IaAEY67F9Q== dependencies: - warcio "^2.3.1" + warcio "^2.4.0" "@xtuc/ieee754@^1.2.0": version "1.2.0" @@ -3750,10 +3750,10 @@ walk-up-path@^3.0.1: resolved "https://registry.yarnpkg.com/walk-up-path/-/walk-up-path-3.0.1.tgz#c8d78d5375b4966c717eb17ada73dbd41490e886" integrity sha512-9YlCL/ynK3CTlrSRrDxZvUauLzAswPCrsaCgilqFevUYpeEW0/3ScEjaa3kbW/T0ghhkEr7mv+fpjqn1Y1YuTA== -warcio@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/warcio/-/warcio-2.3.1.tgz#8ac9de897de1a556161168f2a3938b60929908ca" - integrity sha512-PjcWqzXfs6HdWfHi1V/i8MoMmV5M0Csg3rOa2mqCJ1dmCJXswVfQ0VXbEVumwavNIW2oFFj6LJoCHHeL4Ls/zw== +warcio@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/warcio/-/warcio-2.4.0.tgz#13bae2837f1bbf5cf7585f75857e6311d30557bd" + integrity sha512-EfxXCgsnZ35CGf2j99QBMyB6EI98KEQ6YmeER+8Lnv/4KFJ3thT76PiX37HfZVbPJS21JihA0Eddjk9QBQRlPg== dependencies: "@types/pako" "^1.0.7" "@types/stream-buffers" "^3.0.7" @@ -3762,7 +3762,7 @@ warcio@^2.3.1: pako "^1.0.11" tempy "^3.1.0" uuid-random "^1.3.2" - yargs "^17.6.2" + yargs "^17.7.2" watchpack@^2.4.0: version "2.4.0" @@ -3963,19 +3963,6 @@ yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@^17.6.2: - version "17.6.2" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.6.2.tgz#2e23f2944e976339a1ee00f18c77fedee8332541" - integrity sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw== - dependencies: - cliui "^8.0.1" - escalade "^3.1.1" - get-caller-file "^2.0.5" - require-directory "^2.1.1" - string-width "^4.2.3" - y18n "^5.0.5" - yargs-parser "^21.1.1" - yargs@^17.7.2: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269"