From 49fd1c1ab8fc0f5d38515981869bf0db65cc1aa2 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 17 Sep 2024 14:01:57 +0300 Subject: [PATCH 1/5] cvat-core resets cache if not relevant anymore --- cvat-core/src/frames.ts | 169 ++++++++++++++++--------- cvat-core/src/server-proxy.ts | 12 +- cvat-core/src/server-response-types.ts | 1 + 3 files changed, 119 insertions(+), 63 deletions(-) diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts index 96295af7d57..b1e473adf80 100644 --- a/cvat-core/src/frames.ts +++ b/cvat-core/src/frames.ts @@ -16,6 +16,7 @@ import { FieldUpdateTrigger } from './common'; // frame storage by job id const frameDataCache: Record data.stop_frame, }, + chunksUpdatedDate: { + get: () => data.chunks_updated_date, + }, }), ); } @@ -491,6 +497,37 @@ function getFrameMeta(jobID, frame): SerializedFramesMetaData['frames'][0] { return frameMeta; } +async function refreshJobCacheIfOutdated(jobID: number): Promise { + const cached = frameDataCache[jobID]; + if (!cached) { + throw new Error('Frame data cache is abscent'); + } + + const META_DATA_RELOAD_PERIOD = 1 * 60 * 60 * 1000; // 1 hour + const isOutdated = (Date.now() - cached.fetchTimestamp) > META_DATA_RELOAD_PERIOD; + + if (isOutdated) { + // get metadata again if outdated + const meta = await getFramesMeta('job', jobID, true); + if (new Date(meta.chunksUpdatedDate) > new Date(cached.meta.chunksUpdatedDate)) { + // chunks were re-defined. Existing data not relevant anymore + // currently we only re-write meta, remove all cached frames from provider and clear cached context images + // other parameters (e.g. chunkSize) are not supposed to be changed + cached.meta = meta; + cached.provider.cleanup(Number.MAX_SAFE_INTEGER); + for (const frame of Object.keys(cached.contextCache)) { + for (const image of Object.values(cached.contextCache[+frame].data)) { + // close images to immediate memory release + image.close(); + } + } + cached.contextCache = {}; + } + + cached.fetchTimestamp = Date.now(); + } +} + export function getContextImage(jobID: number, frame: number): Promise> { return new Promise>((resolve, reject) => { if (!(jobID in frameDataCache)) { @@ -498,73 +535,78 @@ export function getContextImage(jobID: number, frame: number): Promise { - if (frameData.latestContextImagesRequest !== requestId) { - reject(frame); - } else if (frame in frameData.contextCache) { - resolve(frameData.contextCache[frame].data); - } else { - frameData.activeContextRequest = serverProxy.frames.getImageContext(jobID, frame) - .then((encodedImages) => decodeContextImages(encodedImages, 0, relatedFiles)); - frameData.activeContextRequest.then((images) => { - const size = Object.values(images) - .reduce((acc, image) => acc + image.width * image.height * 4, 0); - const totalSize = Object.values(frameData.contextCache) - .reduce((acc, item) => acc + item.size, 0); - if (totalSize > 512 * 1024 * 1024) { - const [leastTimestampFrame] = Object.entries(frameData.contextCache) - .sort(([, item1], [, item2]) => item1.timestamp - item2.timestamp)[0]; - delete frameData.contextCache[leastTimestampFrame]; - } - frameData.contextCache[frame] = { - data: images, - timestamp: Date.now(), - size, - }; + refreshJobCacheIfOutdated(jobID).then(() => { + const frameData = frameDataCache[jobID]; + const requestId = frame; + const { startFrame } = frameData; + const { related_files: relatedFiles } = frameData.meta.frames[frame - startFrame]; - if (frameData.latestContextImagesRequest !== requestId) { - reject(frame); - } else { - resolve(images); - } - }).finally(() => { - frameData.activeContextRequest = null; - }); - } - }; - - if (!frameData.activeContextRequest) { - executor(); + if (relatedFiles === 0) { + resolve({}); + } else if (frame in frameData.contextCache) { + resolve(frameData.contextCache[frame].data); } else { - const checkAndExecute = (): void => { - if (frameData.activeContextRequest) { - // if we just execute in finally - // it might raise multiple server requests for context images - // if the promise was pending before and several requests came for the same frame - // all these requests will stuck on "finally" - // and when the promise fullfilled, it will run all the microtasks - // since they all have the same request id, all they will perform in executor() - frameData.activeContextRequest.finally(() => setTimeout(checkAndExecute)); + frameData.latestContextImagesRequest = requestId; + const executor = (): void => { + if (frameData.latestContextImagesRequest !== requestId) { + reject(frame); + } else if (frame in frameData.contextCache) { + resolve(frameData.contextCache[frame].data); } else { - executor(); + frameData.activeContextRequest = serverProxy.frames.getImageContext(jobID, frame) + .then((encodedImages) => decodeContextImages(encodedImages, 0, relatedFiles)); + frameData.activeContextRequest.then((images) => { + const size = Object.values(images) + .reduce((acc, image) => acc + image.width * image.height * 4, 0); + const totalSize = Object.values(frameData.contextCache) + .reduce((acc, item) => acc + item.size, 0); + if (totalSize > 512 * 1024 * 1024) { + const [leastTimestampFrame] = Object.entries(frameData.contextCache) + .sort(([, item1], [, item2]) => item1.timestamp - item2.timestamp)[0]; + delete frameData.contextCache[leastTimestampFrame]; + } + + frameData.contextCache[frame] = { + data: images, + timestamp: Date.now(), + size, + }; + + if (frameData.latestContextImagesRequest !== requestId) { + reject(frame); + } else { + resolve(images); + } + }).finally(() => { + frameData.activeContextRequest = null; + }); } }; - setTimeout(checkAndExecute); + if (!frameData.activeContextRequest) { + executor(); + } else { + const checkAndExecute = (): void => { + if (frameData.activeContextRequest) { + // if we just execute in finally + // it might raise multiple server requests for context images + // if the promise was pending before and several requests came for the same frame + // all these requests will stuck on "finally" + // and when the promise fullfilled, it will run all the microtasks + // since they all have the same request id, all they will perform in executor() + frameData.activeContextRequest.finally(() => setTimeout(checkAndExecute)); + } else { + executor(); + } + }; + + setTimeout(checkAndExecute); + } } - } + }).catch((error: unknown) => { + reject(error); + }); }); } @@ -594,7 +636,9 @@ export async function getFrame( dimension: DimensionType, getChunk: (chunkNumber: number, quality: ChunkQuality) => Promise, ): Promise { - if (!(jobID in frameDataCache)) { + const dataCacheExists = jobID in frameDataCache; + + if (!dataCacheExists) { const blockType = chunkType === 'video' ? BlockType.MP4VIDEO : BlockType.ARCHIVE; const meta = await getFramesMeta('job', jobID); @@ -610,6 +654,7 @@ export async function getFrame( ); frameDataCache[jobID] = { meta, + fetchTimestamp: Date.now(), chunkSize, mode, startFrame, @@ -633,6 +678,8 @@ export async function getFrame( }; } + await refreshJobCacheIfOutdated(jobID); + const frameMeta = getFrameMeta(jobID, frame); frameDataCache[jobID].provider.setRenderSize(frameMeta.width, frameMeta.height); frameDataCache[jobID].decodeForward = isPlaying; @@ -649,7 +696,7 @@ export async function getFrame( }); } -export async function getDeletedFrames(instanceType: 'job' | 'task', id): Promise> { +export async function getDeletedFrames(instanceType: 'job' | 'task', id: number): Promise> { if (instanceType === 'job') { const { meta } = frameDataCache[id]; return meta.deletedFrames; diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 91dc52a7182..767c12dd1fa 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1481,7 +1481,11 @@ async function getMeta(session: 'job' | 'task', id: number): Promise Date: Tue, 17 Sep 2024 14:14:15 +0300 Subject: [PATCH 2/5] Added explanation --- cvat-core/src/frames.ts | 140 +++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 65 deletions(-) diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts index b1e473adf80..a7aa86bae6c 100644 --- a/cvat-core/src/frames.ts +++ b/cvat-core/src/frames.ts @@ -536,77 +536,73 @@ export function getContextImage(jobID: number, frame: number): Promise { - const frameData = frameDataCache[jobID]; - const requestId = frame; - const { startFrame } = frameData; - const { related_files: relatedFiles } = frameData.meta.frames[frame - startFrame]; - - if (relatedFiles === 0) { - resolve({}); - } else if (frame in frameData.contextCache) { - resolve(frameData.contextCache[frame].data); - } else { - frameData.latestContextImagesRequest = requestId; - const executor = (): void => { - if (frameData.latestContextImagesRequest !== requestId) { - reject(frame); - } else if (frame in frameData.contextCache) { - resolve(frameData.contextCache[frame].data); - } else { - frameData.activeContextRequest = serverProxy.frames.getImageContext(jobID, frame) - .then((encodedImages) => decodeContextImages(encodedImages, 0, relatedFiles)); - frameData.activeContextRequest.then((images) => { - const size = Object.values(images) - .reduce((acc, image) => acc + image.width * image.height * 4, 0); - const totalSize = Object.values(frameData.contextCache) - .reduce((acc, item) => acc + item.size, 0); - if (totalSize > 512 * 1024 * 1024) { - const [leastTimestampFrame] = Object.entries(frameData.contextCache) - .sort(([, item1], [, item2]) => item1.timestamp - item2.timestamp)[0]; - delete frameData.contextCache[leastTimestampFrame]; - } - - frameData.contextCache[frame] = { - data: images, - timestamp: Date.now(), - size, - }; + const frameData = frameDataCache[jobID]; + const requestId = frame; + const { startFrame } = frameData; + const { related_files: relatedFiles } = frameData.meta.frames[frame - startFrame]; + + if (relatedFiles === 0) { + resolve({}); + } else if (frame in frameData.contextCache) { + resolve(frameData.contextCache[frame].data); + } else { + frameData.latestContextImagesRequest = requestId; + const executor = (): void => { + if (frameData.latestContextImagesRequest !== requestId) { + reject(frame); + } else if (frame in frameData.contextCache) { + resolve(frameData.contextCache[frame].data); + } else { + frameData.activeContextRequest = serverProxy.frames.getImageContext(jobID, frame) + .then((encodedImages) => decodeContextImages(encodedImages, 0, relatedFiles)); + frameData.activeContextRequest.then((images) => { + const size = Object.values(images) + .reduce((acc, image) => acc + image.width * image.height * 4, 0); + const totalSize = Object.values(frameData.contextCache) + .reduce((acc, item) => acc + item.size, 0); + if (totalSize > 512 * 1024 * 1024) { + const [leastTimestampFrame] = Object.entries(frameData.contextCache) + .sort(([, item1], [, item2]) => item1.timestamp - item2.timestamp)[0]; + delete frameData.contextCache[leastTimestampFrame]; + } - if (frameData.latestContextImagesRequest !== requestId) { - reject(frame); - } else { - resolve(images); - } - }).finally(() => { - frameData.activeContextRequest = null; - }); - } - }; + frameData.contextCache[frame] = { + data: images, + timestamp: Date.now(), + size, + }; - if (!frameData.activeContextRequest) { - executor(); - } else { - const checkAndExecute = (): void => { - if (frameData.activeContextRequest) { - // if we just execute in finally - // it might raise multiple server requests for context images - // if the promise was pending before and several requests came for the same frame - // all these requests will stuck on "finally" - // and when the promise fullfilled, it will run all the microtasks - // since they all have the same request id, all they will perform in executor() - frameData.activeContextRequest.finally(() => setTimeout(checkAndExecute)); + if (frameData.latestContextImagesRequest !== requestId) { + reject(frame); } else { - executor(); + resolve(images); } - }; - - setTimeout(checkAndExecute); + }).finally(() => { + frameData.activeContextRequest = null; + }); } + }; + + if (!frameData.activeContextRequest) { + executor(); + } else { + const checkAndExecute = (): void => { + if (frameData.activeContextRequest) { + // if we just execute in finally + // it might raise multiple server requests for context images + // if the promise was pending before and several requests came for the same frame + // all these requests will stuck on "finally" + // and when the promise fullfilled, it will run all the microtasks + // since they all have the same request id, all they will perform in executor() + frameData.activeContextRequest.finally(() => setTimeout(checkAndExecute)); + } else { + executor(); + } + }; + + setTimeout(checkAndExecute); } - }).catch((error: unknown) => { - reject(error); - }); + } }); } @@ -678,6 +674,20 @@ export async function getFrame( }; } + // basically the following functions may be affected if job cache is outdated + // - getFrame + // - getContextImage + // - getCachedChunks + // And from this idea we should call refreshJobCacheIfOutdated from each one + // Hovewer, following from the order, these methods are usually called + // it may lead to even more confusing behaviour + // + // Usually user first receives frame, then user receives ranges and finally user receives context images + // In this case (extremely rare, but nevertheless possible) user may get context images related to another frame + // - if cache gets outdated after getFrame() call + // - and before getContextImage() call + // - and both calls refer to the same fram that is refreshed honeypot frame + // Thus, it is better to only call `refreshJobCacheIfOutdated` from getFrame() await refreshJobCacheIfOutdated(jobID); const frameMeta = getFrameMeta(jobID, frame); From 73b27f693adc3bcbd52344b5bf67b792679d767a Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 17 Sep 2024 20:31:40 +0300 Subject: [PATCH 3/5] Fixed comment --- cvat-core/src/frames.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts index a7aa86bae6c..9c35b18ee5b 100644 --- a/cvat-core/src/frames.ts +++ b/cvat-core/src/frames.ts @@ -686,7 +686,7 @@ export async function getFrame( // In this case (extremely rare, but nevertheless possible) user may get context images related to another frame // - if cache gets outdated after getFrame() call // - and before getContextImage() call - // - and both calls refer to the same fram that is refreshed honeypot frame + // - and both calls refer to the same fram that is refreshed honeypot frame and this frame has context images // Thus, it is better to only call `refreshJobCacheIfOutdated` from getFrame() await refreshJobCacheIfOutdated(jobID); From 0bedc1e6fe3eb4648b7eec9866c5f65377a09249 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 30 Sep 2024 12:11:41 +0300 Subject: [PATCH 4/5] Minor updates --- cvat-core/src/frames.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts index 8faa69cc5ab..88c0097929c 100644 --- a/cvat-core/src/frames.ts +++ b/cvat-core/src/frames.ts @@ -16,7 +16,7 @@ import { FieldUpdateTrigger } from './common'; // frame storage by job id const frameDataCache: Record { } const META_DATA_RELOAD_PERIOD = 1 * 60 * 60 * 1000; // 1 hour - const isOutdated = (Date.now() - cached.fetchTimestamp) > META_DATA_RELOAD_PERIOD; + const isOutdated = (Date.now() - cached.metaFetchedTimestamp) > META_DATA_RELOAD_PERIOD; if (isOutdated) { // get metadata again if outdated @@ -625,7 +625,7 @@ async function refreshJobCacheIfOutdated(jobID: number): Promise { cached.contextCache = {}; } - cached.fetchTimestamp = Date.now(); + cached.metaFetchedTimestamp = Date.now(); } } @@ -758,7 +758,7 @@ export async function getFrame( frameDataCache[jobID] = { meta, - fetchTimestamp: Date.now(), + metaFetchedTimestamp: Date.now(), chunkSize, mode, startFrame, @@ -796,7 +796,7 @@ export async function getFrame( // In this case (extremely rare, but nevertheless possible) user may get context images related to another frame // - if cache gets outdated after getFrame() call // - and before getContextImage() call - // - and both calls refer to the same fram that is refreshed honeypot frame and this frame has context images + // - and both calls refer to the same frame that is refreshed honeypot frame and this frame has context images // Thus, it is better to only call `refreshJobCacheIfOutdated` from getFrame() await refreshJobCacheIfOutdated(jobID); From 82452995176d111a3d3906b77cbd10c2e764d8dc Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 3 Oct 2024 12:01:39 +0300 Subject: [PATCH 5/5] Removed mocked API --- cvat-core/src/server-proxy.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index b3ea9bd8a53..9e280ecf481 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -1492,11 +1492,7 @@ async function getMeta(session: 'job' | 'task', id: number): Promise