diff --git a/package-lock.json b/package-lock.json index 911a3b9933..c525433442 100644 --- a/package-lock.json +++ b/package-lock.json @@ -216,7 +216,7 @@ "link": true }, "node_modules/@duckduckgo/privacy-reference-tests": { - "resolved": "git+ssh://git@github.com/duckduckgo/privacy-reference-tests.git#2e73221f9b5d872e05199db6b29f140406c909ae", + "resolved": "git+ssh://git@github.com/duckduckgo/privacy-reference-tests.git#bfa5ea412e3b5275e4693673fbd307fa4466d9a8", "license": "Apache-2.0" }, "node_modules/@duckduckgo/tracker-surrogates": { @@ -11293,8 +11293,8 @@ } }, "@duckduckgo/privacy-reference-tests": { - "version": "git+ssh://git@github.com/duckduckgo/privacy-reference-tests.git#2e73221f9b5d872e05199db6b29f140406c909ae", - "from": "@duckduckgo/privacy-reference-tests@github:duckduckgo/privacy-reference-tests#main" + "version": "git+ssh://git@github.com/duckduckgo/privacy-reference-tests.git#bfa5ea412e3b5275e4693673fbd307fa4466d9a8", + "from": "@duckduckgo/privacy-reference-tests@github:@duckduckgo/privacy-reference-tests#main" }, "@duckduckgo/tracker-surrogates": { "version": "git+ssh://git@github.com/duckduckgo/tracker-surrogates.git#abd6067fac9693cc5a43d48931b111ca08cb0d5a", diff --git a/shared/js/background/broken-site-report.js b/shared/js/background/broken-site-report.js index ebd45f4fec..c6d5535575 100644 --- a/shared/js/background/broken-site-report.js +++ b/shared/js/background/broken-site-report.js @@ -88,6 +88,57 @@ const requestCategoryMapping = { 'ignore-user': 'ignoredByUserRequests' } +async function digestMessage (message) { + const msgUint8 = new TextEncoder().encode(message) + const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8) + const hashArray = Array.from(new Uint8Array(hashBuffer)) + const hashHex = hashArray + .map((b) => b.toString(16).padStart(2, '0')) + .join('') + return hashHex +} + +async function computeLastSentDay (urlString) { + const url = new URL(urlString) + // Output time as a string in the format YYYY-MM-DD + const dayOutput = new Date().toISOString().split('T')[0] + + // Use a sha256 hash prefix of the hostname so that we don't store the full hostname + const hash = await digestMessage(url.hostname) + const hostnameHashPrefix = hash.slice(0, 6) + + const reportTimes = settings.getSetting('brokenSiteReportTimes') || {} + const lastSentDay = reportTimes[hostnameHashPrefix] + + // Update existing time + reportTimes[hostnameHashPrefix] = dayOutput + settings.updateSetting('brokenSiteReportTimes', reportTimes) + + return lastSentDay +} + +/** + * Clears any expired broken site report times + * Called by an alarm every hour to remove entries older than 30 days + */ +export async function clearExpiredBrokenSiteReportTimes () { + await settings.ready() + const brokenSiteReports = settings.getSetting('brokenSiteReportTimes') || {} + // Expiry of 30 days + const expiryTime = new Date().getTime() - 30 * 24 * 60 * 60 * 1000 + for (const hashPrefix in brokenSiteReports) { + const reportTime = new Date(brokenSiteReports[hashPrefix]) + if (reportTime.getTime() < expiryTime) { + delete brokenSiteReports[hashPrefix] + } + } + settings.updateSetting('brokenSiteReportTimes', brokenSiteReports) +} + +export async function clearAllBrokenSiteReportTimes () { + settings.updateSetting('brokenSiteReportTimes', {}) +} + /** * Given an optional category and description, create a report for a given Tab instance. * @@ -103,7 +154,7 @@ const requestCategoryMapping = { * @prop {string | undefined} arg.category - optional category * @prop {string | undefined} arg.description - optional description */ -export function breakageReportForTab ({ +export async function breakageReportForTab ({ tab, tds, remoteConfigEtag, remoteConfigVersion, category, description }) { @@ -137,6 +188,7 @@ export function breakageReportForTab ({ const debugFlags = tab.debugFlags.join(',') const errorDescriptions = JSON.stringify(tab.errorDescriptions) const httpErrorCodes = tab.httpErrorCodes.join(',') + const lastSentDay = await computeLastSentDay(tab.url) const brokenSiteParams = new URLSearchParams({ siteUrl, @@ -155,6 +207,7 @@ export function breakageReportForTab ({ brokenSiteParams.append(key, value.join(',')) } + if (lastSentDay) brokenSiteParams.set('lastSentDay', lastSentDay) if (ampUrl) brokenSiteParams.set('ampUrl', ampUrl) if (category) brokenSiteParams.set('category', category) if (debugFlags) brokenSiteParams.set('debugFlags', debugFlags) diff --git a/shared/js/background/events.js b/shared/js/background/events.js index deb83b02ea..accfb11fa6 100644 --- a/shared/js/background/events.js +++ b/shared/js/background/events.js @@ -17,6 +17,7 @@ import { import tdsStorage from './storage/tds' import httpsStorage from './storage/https' import ATB from './atb' +import { clearExpiredBrokenSiteReportTimes } from './broken-site-report' const utils = require('./utils') const experiment = require('./experiments') const settings = require('./settings') @@ -434,7 +435,9 @@ browserWrapper.createAlarm('clearExpiredHTTPSServiceCache', { periodInMinutes: 6 // Rotate the user agent spoofed browserWrapper.createAlarm('rotateUserAgent', { periodInMinutes: 24 * 60 }) // Rotate the sessionKey -browserWrapper.createAlarm('rotateSessionKey', { periodInMinutes: 24 * 60 }) +browserWrapper.createAlarm('rotateSessionKey', { periodInMinutes: 60 }) +// Expire site breakage reports +browserWrapper.createAlarm('clearExpiredBrokenSiteReportTimes', { periodInMinutes: 60 }) browser.alarms.onAlarm.addListener(async alarmEvent => { // Warning: Awaiting in this function doesn't actually wait for the promise to resolve before unblocking the main thread. @@ -461,6 +464,8 @@ browser.alarms.onAlarm.addListener(async alarmEvent => { await utils.resetSessionKey() } else if (alarmEvent.name === REFETCH_ALIAS_ALARM) { fetchAlias() + } else if (alarmEvent.name === 'clearExpiredBrokenSiteReportTimes') { + await clearExpiredBrokenSiteReportTimes() } }) diff --git a/shared/js/background/startup.js b/shared/js/background/startup.js index 8dc2c4c9db..8315badd8e 100644 --- a/shared/js/background/startup.js +++ b/shared/js/background/startup.js @@ -3,6 +3,7 @@ import { NewTabTrackerStats } from './newtab-tracker-stats' import { TrackerStats } from './classes/tracker-stats.js' import httpsStorage from './storage/https' import tdsStorage from './storage/tds' +import { clearExpiredBrokenSiteReportTimes } from './broken-site-report' const utils = require('./utils') const Companies = require('./companies') const experiment = require('./experiments') @@ -69,6 +70,8 @@ export async function onStartup () { showContextMenuAction() } + await clearExpiredBrokenSiteReportTimes() + if (resolveReadyPromise) { resolveReadyPromise() resolveReadyPromise = null diff --git a/unit-test/background/reference-tests/broken-site-reporting-tests.js b/unit-test/background/reference-tests/broken-site-reporting-tests.js index 9b2379075d..5b7d231c2b 100644 --- a/unit-test/background/reference-tests/broken-site-reporting-tests.js +++ b/unit-test/background/reference-tests/broken-site-reporting-tests.js @@ -1,102 +1,146 @@ import Tab from '../../../shared/js/background/classes/tab' -import { breakageReportForTab } from '../../../shared/js/background/broken-site-report' +import { breakageReportForTab, clearAllBrokenSiteReportTimes } from '../../../shared/js/background/broken-site-report' const loadPixel = require('../../../shared/js/background/load') -const testSets = require('@duckduckgo/privacy-reference-tests/broken-site-reporting/tests.json') +const singleTestSets = require('@duckduckgo/privacy-reference-tests/broken-site-reporting/tests.json') +const multipleTestSets = require('@duckduckgo/privacy-reference-tests/broken-site-reporting/multiple_report_tests.json') + +let loadPixelSpy + +async function submitAndValidateReport (report) { + const trackerName = 'Ad Company' + const trackerObj = { + owner: { + name: trackerName, + displayName: trackerName + } + } + const tab = new Tab({ + url: report.siteURL + }) + tab.upgradedHttps = report.wasUpgraded + + const addRequest = (hostname, action, opts = {}) => { + tab.addToTrackers({ + action, + reason: 'reference tests', + sameEntity: false, + sameBaseDomain: false, + redirectUrl: false, + matchedRule: 'reference tests', + matchedRuleException: false, + tracker: trackerObj, + fullTrackerDomain: hostname, + ...opts + }) + } -for (const setName of Object.keys(testSets)) { - const testSet = testSets[setName] + const addRequests = (trackers, f) => { + (trackers || []).forEach(hostname => { + const opts = f(hostname) + const { action } = opts + addRequest(hostname, action, opts) + }) + } - describe(`Broken Site Reporting tests / ${testSet.name} /`, () => { - testSet.tests.forEach(test => { - if (test.exceptPlatforms && test.exceptPlatforms.includes('web-extension')) { - return - } + const addActionRequests = (trackers, action) => { + addRequests(trackers, _ => ({ action })) + } + + addActionRequests(report.blockedTrackers, 'block') + addActionRequests(report.surrogates, 'redirect') + addActionRequests(report.ignoreRequests, 'ignore') + addActionRequests(report.ignoredByUserRequests, 'ignore-user') + addActionRequests(report.adAttributionRequests, 'ad-attribution') + addActionRequests(report.noActionRequests, 'none') + + await breakageReportForTab({ + tab, + tds: report.blocklistVersion, + remoteConfigEtag: report.remoteConfigEtag, + remoteConfigVersion: report.remoteConfigVersion, + category: report.category, + description: report.providedDescription + }) + expect(loadPixelSpy.calls.count()).withContext('Expect only one pixel').toEqual(1) + + const requestURLString = loadPixelSpy.calls.argsFor(0)[0] + loadPixelSpy.calls.reset() - it(test.name, () => { - const loadPixelSpy = spyOn(loadPixel, 'url').and.returnValue(null) + if (report.expectReportURLPrefix) { + expect(requestURLString.startsWith(report.expectReportURLPrefix)).toBe(true) + } - const trackerName = 'Ad Company' - const trackerObj = { - owner: { - name: trackerName, - displayName: trackerName + if (report.expectReportURLParams) { + const requestUrl = new URL(requestURLString) + // we can't use searchParams because those are automatically decoded + const searchItems = requestUrl.search.split('&') + + report.expectReportURLParams.forEach(param => { + if ('value' in param) { + expect(searchItems).toContain(`${param.name}=${param.value}`) + } + if ('matchesCurrentDay' in param) { + const date = new Date() + const day = String(date.getDate()).padStart(2, '0') + const month = String(date.getMonth() + 1).padStart(2, '0') + const year = date.getFullYear() + const dateString = `${year}-${month}-${day}` + expect(searchItems).toContain(`${param.name}=${dateString}`) + } + if ('matches' in param) { + const regex = new RegExp(param.matches) + const fields = searchItems.map(item => item.split('=')) + for (const [key, value] of fields) { + if (key === param.name) { + expect(value).toMatch(regex) } } - const tab = new Tab({ - url: test.siteURL - }) - tab.upgradedHttps = test.wasUpgraded - - const addRequest = (hostname, action, opts = {}) => { - tab.addToTrackers({ - action, - reason: 'reference tests', - sameEntity: false, - sameBaseDomain: false, - redirectUrl: false, - matchedRule: 'reference tests', - matchedRuleException: false, - tracker: trackerObj, - fullTrackerDomain: hostname, - ...opts - }) - } - - const addRequests = (trackers, f) => { - (trackers || []).forEach(hostname => { - const opts = f(hostname) - const { action } = opts - addRequest(hostname, action, opts) - }) + } + if ('present' in param) { + const fields = searchItems.map(item => item.split('=')[0]) + if (param.present) { + expect(fields).withContext(`Expected ${param.name} to be present in ${searchItems}`).toContain(param.name) + } else { + expect(fields).not.toContain(param.name) } - - const addActionRequests = (trackers, action) => { - addRequests(trackers, _ => ({ action })) + } + }) + } +} +function runTests (testSets, supportsMultipleReports = false) { + for (const setName of Object.keys(testSets)) { + const testSet = testSets[setName] + + describe(`Broken Site Reporting tests / ${testSet.name} /`, () => { + for (const test of testSet.tests) { + if (test.exceptPlatforms && test.exceptPlatforms.includes('web-extension')) { + return } - addActionRequests(test.blockedTrackers, 'block') - addActionRequests(test.surrogates, 'redirect') - addActionRequests(test.ignoreRequests, 'ignore') - addActionRequests(test.ignoredByUserRequests, 'ignore-user') - addActionRequests(test.adAttributionRequests, 'ad-attribution') - addActionRequests(test.noActionRequests, 'none') - - breakageReportForTab({ - tab, - tds: test.blocklistVersion, - remoteConfigEtag: test.remoteConfigEtag, - remoteConfigVersion: test.remoteConfigVersion, - category: test.category, - description: test.providedDescription + it(test.name, async () => { + loadPixelSpy = spyOn(loadPixel, 'url').and.returnValue(null) + await clearAllBrokenSiteReportTimes() + if (supportsMultipleReports) { + for (const report of test.reports) { + await submitAndValidateReport(report) + } + } else { + await submitAndValidateReport(test) + } }) - - expect(loadPixelSpy.calls.count()).toEqual(1) - - const requestURLString = loadPixelSpy.calls.argsFor(0)[0] - - if (test.expectReportURLPrefix) { - expect(requestURLString.startsWith(test.expectReportURLPrefix)).toBe(true) - } - - if (test.expectReportURLParams) { - const requestUrl = new URL(requestURLString) - // we can't use searchParams because those are automatically decoded - const searchItems = requestUrl.search.split('&') - - test.expectReportURLParams.forEach(param => { - expect(searchItems).toContain(`${param.name}=${param.value}`) - }) - } - }) + } }) - }) + } } +runTests(singleTestSets) +runTests(multipleTestSets, true) + describe('Broken Site Reporting tests / protections state', () => { - function submit (tab) { - const loadPixelSpy = spyOn(loadPixel, 'url').and.returnValue(null) - breakageReportForTab({ + async function submit (tab) { + loadPixelSpy = spyOn(loadPixel, 'url').and.returnValue(null) + await breakageReportForTab({ tab, tds: 'abc123', remoteConfigEtag: 'abd142', @@ -107,46 +151,46 @@ describe('Broken Site Reporting tests / protections state', () => { const requestURLString = loadPixelSpy.calls.argsFor(0)[0] return new URL(requestURLString).searchParams } - it('sends 1 when protections are enabled', () => { + it('sends 1 when protections are enabled', async () => { const tab = new Tab({ url: 'https://example.com' }) spyOnProperty(tab.site, 'enabledFeatures').and.returnValue(['contentBlocking']) - const params = submit(tab) + const params = await submit(tab) expect(params.get('protectionsState')).toEqual('1') }) - it('sends 1 when site is denylisted', () => { + it('sends 1 when site is denylisted', async () => { const tab = new Tab({ url: 'https://example.com' }) spyOnProperty(tab.site, 'enabledFeatures').and.returnValue([]) spyOnProperty(tab.site, 'denylisted').and.returnValue(true) - const params = submit(tab) + const params = await submit(tab) expect(params.get('protectionsState')).toEqual('1') }) - it('sends 0 when site is allowlisted', () => { + it('sends 0 when site is allowlisted', async () => { const tab = new Tab({ url: 'https://example.com' }) spyOnProperty(tab.site, 'enabledFeatures').and.returnValue(['contentBlocking']) spyOnProperty(tab.site, 'allowlisted').and.returnValue(true) - const params = submit(tab) + const params = await submit(tab) expect(params.get('protectionsState')).toEqual('0') }) - it('sends 0 when contentBlocking is not enabled', () => { + it('sends 0 when contentBlocking is not enabled', async () => { const tab = new Tab({ url: 'https://example.com' }) // missing `contentBlocking` spyOnProperty(tab.site, 'enabledFeatures').and.returnValue([]) - const params = submit(tab) + const params = await submit(tab) expect(params.get('protectionsState')).toEqual('0') }) - it('sends 0 when domain is in unprotectedTemporary', () => { + it('sends 0 when domain is in unprotectedTemporary', async () => { const tab = new Tab({ url: 'https://example.com' }) spyOnProperty(tab.site, 'enabledFeatures').and.returnValue(['contentBlocking']) spyOnProperty(tab.site, 'isBroken').and.returnValue(true) - const params = submit(tab) + const params = await submit(tab) expect(params.get('protectionsState')).toEqual('0') }) })