From e2de68c67bca759300cfac12f4fcf3738de1b63b Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 15 Mar 2024 18:38:02 -0300 Subject: [PATCH 1/4] feat: Http cache stats --- lib/modules/datasource/npm/get.ts | 8 ++- lib/util/http/index.ts | 8 ++- lib/util/stats.spec.ts | 78 +++++++++++++++++++++++++ lib/util/stats.ts | 96 +++++++++++++++++++++++++++++++ lib/workers/repository/index.ts | 8 ++- 5 files changed, 195 insertions(+), 3 deletions(-) diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts index 33f5da96829bda..774ffcca18a0cf 100644 --- a/lib/modules/datasource/npm/get.ts +++ b/lib/modules/datasource/npm/get.ts @@ -7,12 +7,13 @@ import { HOST_DISABLED } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import * as packageCache from '../../../util/cache/package'; -import type { Http } from '../../../util/http'; +import { Http } from '../../../util/http'; import type { HttpOptions } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; import { joinUrlParts } from '../../../util/url'; import type { Release, ReleaseResult } from '../types'; import type { CachedReleaseResult, NpmResponse } from './types'; +import { HttpCacheStats } from '../../../util/stats'; export const CACHE_REVISION = 1; @@ -91,10 +92,13 @@ export async function getDependency( ); if (softExpireAt.isValid && softExpireAt > DateTime.local()) { logger.trace('Cached result is not expired - reusing'); + HttpCacheStats.incLocalHits(packageUrl); delete cachedResult.cacheData; return cachedResult; } + logger.trace('Cached result is soft expired'); + HttpCacheStats.incLocalMisses(packageUrl); } else { logger.trace( `Package cache for npm package "${packageName}" is from an old revision - discarding`, @@ -127,6 +131,7 @@ export async function getDependency( const raw = await http.getJson(packageUrl, options); if (cachedResult?.cacheData && raw.statusCode === 304) { logger.trace(`Cached npm result for ${packageName} is revalidated`); + HttpCacheStats.incRemoteHits(packageUrl); cachedResult.cacheData.softExpireAt = softExpireAt; await packageCache.set( cacheNamespace, @@ -137,6 +142,7 @@ export async function getDependency( delete cachedResult.cacheData; return cachedResult; } + HttpCacheStats.incRemoteMisses(packageUrl); const etag = raw.headers.etag; const res = raw.body; if (!res.versions || !Object.keys(res.versions).length) { diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 107f4726344928..d6d46cd09935bf 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -11,7 +11,11 @@ import { getCache } from '../cache/repository'; import { clone } from '../clone'; import { hash } from '../hash'; import { type AsyncResult, Result } from '../result'; -import { type HttpRequestStatsDataPoint, HttpStats } from '../stats'; +import { + type HttpRequestStatsDataPoint, + HttpStats, + HttpCacheStats, +} from '../stats'; import { resolveBaseUrl } from '../url'; import { applyAuthorization, removeAuthorization } from './auth'; import { hooks } from './hooks'; @@ -279,6 +283,7 @@ export class Http { logger.debug( `http cache: saving ${url} (etag=${resCopy.headers.etag}, lastModified=${resCopy.headers['last-modified']})`, ); + HttpCacheStats.incRemoteMisses(url); cache.httpCache[url] = { etag: resCopy.headers.etag, httpResponse: copyResponse(res, deepCopyNeeded), @@ -290,6 +295,7 @@ export class Http { logger.debug( `http cache: Using cached response: ${url} from ${cache.httpCache[url].timeStamp}`, ); + HttpCacheStats.incRemoteHits(url); const cacheCopy = copyResponse( cache.httpCache[url].httpResponse, deepCopyNeeded, diff --git a/lib/util/stats.spec.ts b/lib/util/stats.spec.ts index f23bfe35d6e3a1..64dc0bb7ea174c 100644 --- a/lib/util/stats.spec.ts +++ b/lib/util/stats.spec.ts @@ -1,6 +1,7 @@ import { logger } from '../../test/util'; import * as memCache from './cache/memory'; import { + HttpCacheStats, HttpStats, LookupStats, PackageCacheStats, @@ -455,4 +456,81 @@ describe('util/stats', () => { }); }); }); + + describe('HttpCacheStats', () => { + it('returns empty data', () => { + const res = HttpCacheStats.getData(); + expect(res).toEqual({}); + }); + + it('ignores wrong url', () => { + HttpCacheStats.incLocalHits(''); + expect(HttpCacheStats.getData()).toEqual({}); + }); + + it('writes data points', () => { + HttpCacheStats.incLocalHits('https://example.com/foo'); + HttpCacheStats.incLocalHits('https://example.com/foo'); + HttpCacheStats.incLocalMisses('https://example.com/foo'); + HttpCacheStats.incLocalMisses('https://example.com/bar'); + HttpCacheStats.incRemoteHits('https://example.com/bar'); + HttpCacheStats.incRemoteMisses('https://example.com/bar'); + + const res = HttpCacheStats.getData(); + + expect(res).toEqual({ + 'https://example.com/bar': { + localHits: 0, + localMisses: 1, + localTotal: 1, + remoteHits: 1, + remoteMisses: 1, + remoteTotal: 2, + }, + 'https://example.com/foo': { + localHits: 2, + localMisses: 1, + localTotal: 3, + remoteHits: 0, + remoteMisses: 0, + remoteTotal: 0, + }, + }); + }); + + it('prints report', () => { + HttpCacheStats.incLocalHits('https://example.com/foo'); + HttpCacheStats.incLocalHits('https://example.com/foo'); + HttpCacheStats.incLocalMisses('https://example.com/foo'); + HttpCacheStats.incLocalMisses('https://example.com/bar'); + HttpCacheStats.incRemoteHits('https://example.com/bar'); + HttpCacheStats.incRemoteMisses('https://example.com/bar'); + + HttpCacheStats.report(); + + expect(logger.logger.debug).toHaveBeenCalledTimes(1); + const [data, msg] = logger.logger.debug.mock.calls[0]; + expect(msg).toBe('HTTP cache statistics'); + expect(data).toEqual( + { + "https://example.com/bar": { + "localHits": 0, + "localMisses": 1, + "localTotal": 1, + "remoteHits": 1, + "remoteMisses": 1, + "remoteTotal": 2, + }, + "https://example.com/foo": { + "localHits": 2, + "localMisses": 1, + "localTotal": 3, + "remoteHits": 0, + "remoteMisses": 0, + "remoteTotal": 0, + }, + } + ); + }); + }); }); diff --git a/lib/util/stats.ts b/lib/util/stats.ts index 6508f49d834551..c0e6996e360db5 100644 --- a/lib/util/stats.ts +++ b/lib/util/stats.ts @@ -235,3 +235,99 @@ export class HttpStats { logger.debug({ urls, hosts, requests }, 'HTTP statistics'); } } + +interface HttpCacheHostStatsData { + localHits: number; + localMisses: number; + localTotal: number; + remoteHits: number; + remoteMisses: number; + remoteTotal: number; +} + +type HttpCacheStatsData = Record; + +export class HttpCacheStats { + static getData(): HttpCacheStatsData { + return memCache.get('http-cache-stats') ?? {}; + } + + static read(key: string): HttpCacheHostStatsData { + return ( + this.getData()?.[key] ?? { + localHits: 0, + localMisses: 0, + localTotal: 0, + remoteHits: 0, + remoteMisses: 0, + remoteTotal: 0, + } + ); + } + + static write(key: string, data: HttpCacheHostStatsData): void { + const stats = memCache.get('http-cache-stats') ?? {}; + stats[key] = data; + memCache.set('http-cache-stats', stats); + } + + static getBaseUrl(url: string): string | null { + const parsedUrl = parseUrl(url); + if (!parsedUrl) { + logger.debug({ url }, 'Failed to parse URL during cache stats'); + return null; + } + const { origin, pathname } = parsedUrl; + const baseUrl = `${origin}${pathname}`; + return baseUrl; + } + + static incLocalHits(url: string): void { + const baseUrl = HttpCacheStats.getBaseUrl(url); + if (baseUrl) { + const host = baseUrl; + const stats = HttpCacheStats.read(host); + stats.localHits += 1; + stats.localTotal += 1; + HttpCacheStats.write(host, stats); + } + } + + static incLocalMisses(url: string): void { + const baseUrl = HttpCacheStats.getBaseUrl(url); + if (baseUrl) { + const host = baseUrl; + const stats = HttpCacheStats.read(host); + stats.localMisses += 1; + stats.localTotal += 1; + HttpCacheStats.write(host, stats); + } + } + + static incRemoteHits(url: string): void { + const baseUrl = HttpCacheStats.getBaseUrl(url); + if (baseUrl) { + const host = baseUrl; + const stats = HttpCacheStats.read(host); + stats.remoteHits += 1; + stats.remoteTotal += 1; + HttpCacheStats.write(host, stats); + } + } + + static incRemoteMisses(url: string): void { + const baseUrl = HttpCacheStats.getBaseUrl(url); + if (baseUrl) { + const host = baseUrl; + const stats = HttpCacheStats.read(host); + stats.remoteMisses += 1; + stats.remoteTotal += 1; + HttpCacheStats.write(host, stats); + } + } + + static report(): void { + const stats = HttpCacheStats.getData(); + logger.debug(stats, 'HTTP cache statistics'); + } +} diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts index 01c0658eb3e79b..fa7fa880e833eb 100644 --- a/lib/workers/repository/index.ts +++ b/lib/workers/repository/index.ts @@ -19,7 +19,12 @@ import { clearDnsCache, printDnsStats } from '../../util/http/dns'; import * as queue from '../../util/http/queue'; import * as throttle from '../../util/http/throttle'; import { addSplit, getSplits, splitInit } from '../../util/split'; -import { HttpStats, LookupStats, PackageCacheStats } from '../../util/stats'; +import { + HttpCacheStats, + HttpStats, + LookupStats, + PackageCacheStats, +} from '../../util/stats'; import { setBranchCache } from './cache'; import { extractRepoProblems } from './common'; import { ensureDependencyDashboard } from './dependency-dashboard'; @@ -126,6 +131,7 @@ export async function renovateRepository( logger.debug(splits, 'Repository timing splits (milliseconds)'); PackageCacheStats.report(); HttpStats.report(); + HttpCacheStats.report(); LookupStats.report(); printDnsStats(); clearDnsCache(); From f2371d65c2f4bb45c9354de3bfe2dc793d181eeb Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 15 Mar 2024 18:41:36 -0300 Subject: [PATCH 2/4] Fix prettier --- lib/util/stats.spec.ts | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/lib/util/stats.spec.ts b/lib/util/stats.spec.ts index 64dc0bb7ea174c..4ea3c275855af6 100644 --- a/lib/util/stats.spec.ts +++ b/lib/util/stats.spec.ts @@ -511,26 +511,24 @@ describe('util/stats', () => { expect(logger.logger.debug).toHaveBeenCalledTimes(1); const [data, msg] = logger.logger.debug.mock.calls[0]; expect(msg).toBe('HTTP cache statistics'); - expect(data).toEqual( - { - "https://example.com/bar": { - "localHits": 0, - "localMisses": 1, - "localTotal": 1, - "remoteHits": 1, - "remoteMisses": 1, - "remoteTotal": 2, - }, - "https://example.com/foo": { - "localHits": 2, - "localMisses": 1, - "localTotal": 3, - "remoteHits": 0, - "remoteMisses": 0, - "remoteTotal": 0, - }, - } - ); + expect(data).toEqual({ + 'https://example.com/bar': { + localHits: 0, + localMisses: 1, + localTotal: 1, + remoteHits: 1, + remoteMisses: 1, + remoteTotal: 2, + }, + 'https://example.com/foo': { + localHits: 2, + localMisses: 1, + localTotal: 3, + remoteHits: 0, + remoteMisses: 0, + remoteTotal: 0, + }, + }); }); }); }); From 529dae91a7fcf9429ee0aae5503f477d832004db Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 15 Mar 2024 18:44:44 -0300 Subject: [PATCH 3/4] Fix eslint --- lib/modules/datasource/npm/get.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts index 774ffcca18a0cf..0848f5fdfc01d6 100644 --- a/lib/modules/datasource/npm/get.ts +++ b/lib/modules/datasource/npm/get.ts @@ -7,13 +7,13 @@ import { HOST_DISABLED } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { ExternalHostError } from '../../../types/errors/external-host-error'; import * as packageCache from '../../../util/cache/package'; -import { Http } from '../../../util/http'; +import type { Http } from '../../../util/http'; import type { HttpOptions } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; +import { HttpCacheStats } from '../../../util/stats'; import { joinUrlParts } from '../../../util/url'; import type { Release, ReleaseResult } from '../types'; import type { CachedReleaseResult, NpmResponse } from './types'; -import { HttpCacheStats } from '../../../util/stats'; export const CACHE_REVISION = 1; From 2d4602cfdbabb3421e965b6b4422019e03f0ef0e Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Sat, 16 Mar 2024 09:30:14 -0300 Subject: [PATCH 4/4] Fix eslint --- lib/util/http/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index d6d46cd09935bf..d6139238eff4d7 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -12,9 +12,9 @@ import { clone } from '../clone'; import { hash } from '../hash'; import { type AsyncResult, Result } from '../result'; import { + HttpCacheStats, type HttpRequestStatsDataPoint, HttpStats, - HttpCacheStats, } from '../stats'; import { resolveBaseUrl } from '../url'; import { applyAuthorization, removeAuthorization } from './auth';