From 9c053be6024db127f1d19dafa7353cf71589eaea Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 16:38:34 -0500 Subject: [PATCH 01/11] chore: add ua-parser-js dependency to generator-networks-creator --- package-lock.json | 122 +++++++++++++++++- .../generator-networks-creator/package.json | 3 +- 2 files changed, 122 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 59798a2d..2a459a07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3366,6 +3366,26 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-europe-js": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/detect-europe-js/-/detect-europe-js-0.1.2.tgz", + "integrity": "sha512-lgdERlL3u0aUdHocoouzT10d9I89VVhk0qNRmll7mXdGfJT1/wqZ2ZLA4oJAjeACPY5fT1wsbq2AT+GkuInsow==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ], + "license": "MIT" + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -5468,6 +5488,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-standalone-pwa": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-standalone-pwa/-/is-standalone-pwa-0.1.1.tgz", + "integrity": "sha512-9Cbovsa52vNQCjdXOzeQq5CnCbAcRk05aU62K20WO372NrTv0NxibLFCK6lQ4/iZEFdEA3p3t2VNOn8AJ53F5g==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ], + "license": "MIT" + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -9102,6 +9142,57 @@ "node": ">=14.17" } }, + "node_modules/ua-is-frozen": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ua-is-frozen/-/ua-is-frozen-0.1.2.tgz", + "integrity": "sha512-RwKDW2p3iyWn4UbaxpP2+VxwqXh0jpvdxsYpZ5j/MLLiQOfbsV5shpgQiw93+KMYQPcteeMQ289MaAFzs3G9pw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ], + "license": "MIT" + }, + "node_modules/ua-parser-js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-2.0.0.tgz", + "integrity": "sha512-SASgD4RlB7+SCMmlVNqrhPw0f/2pGawWBzJ2+LwGTD0GgNnrKGzPJDiraGHJDwW9Zm5DH2lTmUpqDpbZjJY4+Q==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + }, + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" + } + ], + "license": "AGPL-3.0-or-later", + "dependencies": { + "detect-europe-js": "^0.1.2", + "is-standalone-pwa": "^0.1.1", + "ua-is-frozen": "^0.1.2" + }, + "bin": { + "ua-parser-js": "script/cli.js" + }, + "engines": { + "node": "*" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", @@ -9596,7 +9687,8 @@ "dependencies": { "generative-bayesian-network": "^2.1.62", "node-fetch": "^2.6.7", - "tslib": "^2.4.0" + "tslib": "^2.4.0", + "ua-parser-js": "^2.0.0" }, "engines": { "node": ">=16.0.0" @@ -12155,6 +12247,11 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, + "detect-europe-js": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/detect-europe-js/-/detect-europe-js-0.1.2.tgz", + "integrity": "sha512-lgdERlL3u0aUdHocoouzT10d9I89VVhk0qNRmll7mXdGfJT1/wqZ2ZLA4oJAjeACPY5fT1wsbq2AT+GkuInsow==" + }, "detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -13171,7 +13268,8 @@ "requires": { "generative-bayesian-network": "^2.1.62", "node-fetch": "^2.6.7", - "tslib": "^2.4.0" + "tslib": "^2.4.0", + "ua-parser-js": "^2.0.0" } }, "gensync": { @@ -13736,6 +13834,11 @@ "call-bind": "^1.0.2" } }, + "is-standalone-pwa": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-standalone-pwa/-/is-standalone-pwa-0.1.1.tgz", + "integrity": "sha512-9Cbovsa52vNQCjdXOzeQq5CnCbAcRk05aU62K20WO372NrTv0NxibLFCK6lQ4/iZEFdEA3p3t2VNOn8AJ53F5g==" + }, "is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -16393,6 +16496,21 @@ "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", "dev": true }, + "ua-is-frozen": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ua-is-frozen/-/ua-is-frozen-0.1.2.tgz", + "integrity": "sha512-RwKDW2p3iyWn4UbaxpP2+VxwqXh0jpvdxsYpZ5j/MLLiQOfbsV5shpgQiw93+KMYQPcteeMQ289MaAFzs3G9pw==" + }, + "ua-parser-js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-2.0.0.tgz", + "integrity": "sha512-SASgD4RlB7+SCMmlVNqrhPw0f/2pGawWBzJ2+LwGTD0GgNnrKGzPJDiraGHJDwW9Zm5DH2lTmUpqDpbZjJY4+Q==", + "requires": { + "detect-europe-js": "^0.1.2", + "is-standalone-pwa": "^0.1.1", + "ua-is-frozen": "^0.1.2" + } + }, "unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", diff --git a/packages/generator-networks-creator/package.json b/packages/generator-networks-creator/package.json index a1e03d7d..9a168dd5 100755 --- a/packages/generator-networks-creator/package.json +++ b/packages/generator-networks-creator/package.json @@ -19,7 +19,8 @@ "dependencies": { "generative-bayesian-network": "^2.1.62", "node-fetch": "^2.6.7", - "tslib": "^2.4.0" + "tslib": "^2.4.0", + "ua-parser-js": "^2.0.0" }, "repository": { "type": "git", From 0b68cccae14b0cdaf60ab5793872a1ad10bf319b Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 16:40:15 -0500 Subject: [PATCH 02/11] feat: Implement the possible rigid checks on the fingerprint given the record structure --- .../src/generator-networks-creator.ts | 318 ++++++++++++++---- 1 file changed, 246 insertions(+), 72 deletions(-) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index b245cd5b..b54e9764 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -1,8 +1,9 @@ import fs from 'fs'; -import path from 'path'; +import path, { parse } from 'path'; import { BayesianNetwork } from 'generative-bayesian-network'; import fetch from 'node-fetch'; +import { UAParser } from 'ua-parser-js'; const browserHttpNodeName = '*BROWSER_HTTP'; const httpVersionNodeName = '*HTTP_VERSION'; @@ -20,42 +21,119 @@ const nonGeneratedNodes = [ const STRINGIFIED_PREFIX = '*STRINGIFIED*'; -const PLUGIN_CHARACTERISTICS_ATTRIBUTES = [ - 'plugins', - 'mimeTypes', -]; - -async function prepareRecords(records: Record[], preprocessingType: string) : Promise[]> { - const cleanedRecords = records - .filter(( - { - requestFingerprint: { headers }, - browserFingerprint, - }) => { - return (headers['user-agent'] ?? headers['User-Agent']) === browserFingerprint.userAgent; - }) - .filter( - ({ - browserFingerprint: { - screen: { width, height }, - userAgent, - }, - }) => ((width >= 1280 && width > height) || (width < height && /phone|android|mobile/i.test(userAgent))), - ) - .map((record) => ({ ...record, userAgent: record.browserFingerprint.userAgent } as any)); +const PLUGIN_CHARACTERISTICS_ATTRIBUTES = ['plugins', 'mimeTypes']; + +async function prepareRecords( + records: Record[], + preprocessingType: string +): Promise[]> { + const cleanedRecords = []; + + for (const record of records) { + const { + requestFingerprint: { headers }, + browserFingerprint: fingerprint, + } = record; + + // The webdriver attribute should not be truthy + if (fingerprint.webdriver) continue; + + const validPluginAndMime = + 'plugins' in fingerprint && + 'mimeTypes' in fingerprint && + fingerprint.plugins.length > 0 && + fingerprint.mimeTypes.length > 0; + + // The plugins and mimeTypes should be present and non-empty + if (!validPluginAndMime) continue; + + const validUserAgent = + fingerprint.userAgent === + (headers['user-agent'] ?? headers['User-Agent']); + + // The userAgent should match the one in the headers + if (!validUserAgent) continue; + + const validUserAgentData = + !('userAgentData' in fingerprint) || + ('brands' in fingerprint.userAgentData && + 'mobile' in fingerprint.userAgentData && + 'platform' in fingerprint.userAgentData && + fingerprint.userAgentData.brands.length === 3); + + // The userAgentData should have the correct structure + if (!validUserAgentData) continue; + + const validLanguage = + fingerprint.language && + 'languages' in fingerprint && + fingerprint.languages.length > 0 && + fingerprint.language === fingerprint.languages[0]; + + // The language should be the first in the list + if (!validLanguage) continue; + + const parsedUserAgent = await UAParser( + fingerprint.userAgent, + headers + ).withClientHints(); + + const validBrowser = + parsedUserAgent.browser.name && + [ + 'Edge', + 'Chrome', + 'Chrome Mobile', + 'Firefox', + 'Safari', + 'Safari Mobile', + ].includes(parsedUserAgent.browser.name); + + // The browser should be one of the supported ones + if (!validBrowser) continue; + + const desktopFingerprint = + parsedUserAgent.device.type === undefined || + !['wearable', 'mobile'].includes(parsedUserAgent.device.type); + + const validDeviceType = + parsedUserAgent.device.type === 'mobile' || + parsedUserAgent.device.type === 'tablet' || + desktopFingerprint; + + // The device type should be mobile, tablet or desktop + if (!validDeviceType) continue; + + const validTouchSupport = + desktopFingerprint || fingerprint.userAgentData?.mobile !== true + ? fingerprint.maxTouchPoints === 0 + : fingerprint.maxTouchPoints > 0; + + // The maxTouchPoints should be 0 for desktops and > 0 for mobile devices + if (!validTouchSupport) continue; + + cleanedRecords.push({ + ...record, + userAgent: record.browserFingerprint.userAgent, + } as any); + } // TODO this could break if the list is not there anymore // The robots list is available under the MIT license, for details see https://github.com/atmire/COUNTER-Robots/blob/master/LICENSE - const robotUserAgents = await fetch('https://raw.githubusercontent.com/atmire/COUNTER-Robots/master/COUNTER_Robots_list.json') - .then(async (res) => res.json()) as {pattern: string}[]; + const robotUserAgents = (await fetch( + 'https://raw.githubusercontent.com/atmire/COUNTER-Robots/master/COUNTER_Robots_list.json' + ).then(async (res) => res.json())) as { pattern: string }[]; const deconstructedRecords = []; const userAgents = new Set(); for (let x = 0; x < cleanedRecords.length; x++) { let record = cleanedRecords[x]; const { userAgent } = record as { userAgent: string }; - let useRecord = !userAgent.match(/(bot|bots|slurp|spider|crawler|crawl)\b/i) - && !robotUserAgents.some((robot) => userAgent.match(new RegExp(robot.pattern, 'i'))); + let useRecord = + !userAgent.match(/(bot|bots|slurp|spider|crawler|crawl)\b/i) && + !robotUserAgents.some((robot) => + userAgent.match(new RegExp(robot.pattern, 'i')) + ); if (useRecord) { if (preprocessingType === 'headers') { @@ -70,7 +148,9 @@ async function prepareRecords(records: Record[], preprocessingType: } } - if (useRecord) { deconstructedRecords.push(record); } else { + if (useRecord) { + deconstructedRecords.push(record); + } else { userAgents.add(userAgent); } } @@ -97,8 +177,12 @@ async function prepareRecords(records: Record[], preprocessingType: return reorganizedRecords; } + export class GeneratorNetworksCreator { - private getDeviceOS(userAgent: string) : { device: string; operatingSystem: string } { + private getDeviceOS(userAgent: string): { + device: string; + operatingSystem: string; + } { let operatingSystem = missingValueDatasetToken; if (/windows/i.test(userAgent)) { operatingSystem = 'windows'; @@ -120,7 +204,9 @@ export class GeneratorNetworksCreator { return { device, operatingSystem }; } - private getBrowserNameVersion(userAgent: string) : `${string}/${string}` | typeof missingValueDatasetToken { + private getBrowserNameVersion( + userAgent: string + ): `${string}/${string}` | typeof missingValueDatasetToken { const canonicalNames = { chrome: 'chrome', crios: 'chrome', @@ -133,10 +219,12 @@ export class GeneratorNetworksCreator { edgios: 'edge', } as Record; - const unsupportedBrowsers = /opr|yabrowser|SamsungBrowser|UCBrowser|vivaldi/i; + const unsupportedBrowsers = + /opr|yabrowser|SamsungBrowser|UCBrowser|vivaldi/i; const edge = /(edg(a|ios|e)?)\/([0-9.]*)/i; const safari = /Version\/([\d.]+)( Mobile\/[a-z0-9]+)? Safari/i; - const supportedBrowsers = /(firefox|fxios|chrome|crios|safari)\/([0-9.]*)/i; + const supportedBrowsers = + /(firefox|fxios|chrome|crios|safari)\/([0-9.]*)/i; if (unsupportedBrowsers.test(userAgent)) { return missingValueDatasetToken; @@ -160,25 +248,52 @@ export class GeneratorNetworksCreator { return missingValueDatasetToken; } - async prepareHeaderGeneratorFiles(datasetPath: string, resultsPath: string) { + async prepareHeaderGeneratorFiles( + datasetPath: string, + resultsPath: string + ) { const datasetText = fs.readFileSync(datasetPath, { encoding: 'utf8' }); - const records = await prepareRecords(JSON.parse(datasetText), 'headers'); - - const inputGeneratorNetwork = new BayesianNetwork({ path: path.join(__dirname, 'network_structures', 'input-network-structure.zip') }); - const headerGeneratorNetwork = new BayesianNetwork({ path: path.join(__dirname, 'network_structures', 'header-network-structure.zip') }); + const records = await prepareRecords( + JSON.parse(datasetText), + 'headers' + ); + + const inputGeneratorNetwork = new BayesianNetwork({ + path: path.join( + __dirname, + 'network_structures', + 'input-network-structure.zip' + ), + }); + const headerGeneratorNetwork = new BayesianNetwork({ + path: path.join( + __dirname, + 'network_structures', + 'header-network-structure.zip' + ), + }); // eslint-disable-next-line dot-notation - const desiredHeaderAttributes = Object.keys(headerGeneratorNetwork['nodesByName']) - .filter((attribute) => !nonGeneratedNodes.includes(attribute)); + const desiredHeaderAttributes = Object.keys( + headerGeneratorNetwork['nodesByName'] + ).filter((attribute) => !nonGeneratedNodes.includes(attribute)); let selectedRecords = records.map((record) => { - return Object.entries(record).reduce((acc: typeof record, [key, value]) => { - if (desiredHeaderAttributes.includes(key)) acc[key] = value ?? missingValueDatasetToken; - return acc; - }, {}); + return Object.entries(record).reduce( + (acc: typeof record, [key, value]) => { + if (desiredHeaderAttributes.includes(key)) + acc[key] = value ?? missingValueDatasetToken; + return acc; + }, + {} + ); }); selectedRecords = selectedRecords.map((record) => { - const userAgent = (record['user-agent'] !== missingValueDatasetToken ? record['user-agent'] : record['User-Agent']).toLowerCase(); + const userAgent = ( + record['user-agent'] !== missingValueDatasetToken + ? record['user-agent'] + : record['User-Agent'] + ).toLowerCase(); const browser = this.getBrowserNameVersion(userAgent); const { device, operatingSystem } = this.getDeviceOS(userAgent); @@ -188,72 +303,131 @@ export class GeneratorNetworksCreator { [browserNodeName]: browser, [operatingSystemNodeName]: operatingSystem, [deviceNodeName]: device, - [browserHttpNodeName]: `${browser}|${(record[httpVersionNodeName] as string).startsWith('_1') ? '1' : '2'}`, + [browserHttpNodeName]: `${browser}|${ + (record[httpVersionNodeName] as string).startsWith('_1') + ? '1' + : '2' + }`, }; }); headerGeneratorNetwork.setProbabilitiesAccordingToData(selectedRecords); inputGeneratorNetwork.setProbabilitiesAccordingToData(selectedRecords); - const inputNetworkDefinitionPath = path.join(resultsPath, 'input-network-definition.zip'); - const headerNetworkDefinitionPath = path.join(resultsPath, 'header-network-definition.zip'); - const browserHelperFilePath = path.join(resultsPath, 'browser-helper-file.json'); - - headerGeneratorNetwork.saveNetworkDefinition({ path: headerNetworkDefinitionPath }); - inputGeneratorNetwork.saveNetworkDefinition({ path: inputNetworkDefinitionPath }); + const inputNetworkDefinitionPath = path.join( + resultsPath, + 'input-network-definition.zip' + ); + const headerNetworkDefinitionPath = path.join( + resultsPath, + 'header-network-definition.zip' + ); + const browserHelperFilePath = path.join( + resultsPath, + 'browser-helper-file.json' + ); + + headerGeneratorNetwork.saveNetworkDefinition({ + path: headerNetworkDefinitionPath, + }); + inputGeneratorNetwork.saveNetworkDefinition({ + path: inputNetworkDefinitionPath, + }); - const uniqueBrowsersAndHttps = Array.from(new Set(selectedRecords.map((record) => record[browserHttpNodeName]))); - fs.writeFileSync(browserHelperFilePath, JSON.stringify(uniqueBrowsersAndHttps)); + const uniqueBrowsersAndHttps = Array.from( + new Set( + selectedRecords.map((record) => record[browserHttpNodeName]) + ) + ); + fs.writeFileSync( + browserHelperFilePath, + JSON.stringify(uniqueBrowsersAndHttps) + ); } - async prepareFingerprintGeneratorFiles(datasetPath: string, resultsPath: string) { - const datasetText = fs.readFileSync(datasetPath, { encoding: 'utf8' }).replace(/^\ufeff/, ''); - const records = await prepareRecords(JSON.parse(datasetText), 'fingerprints'); + async prepareFingerprintGeneratorFiles( + datasetPath: string, + resultsPath: string + ) { + const datasetText = fs + .readFileSync(datasetPath, { encoding: 'utf8' }) + .replace(/^\ufeff/, ''); + const records = await prepareRecords( + JSON.parse(datasetText), + 'fingerprints' + ); for (let x = 0; x < records.length; x++) { // eslint-disable-next-line no-console - if (x % 1000 === 0) console.log(`Processing record ${x} of ${records.length}`); + if (x % 1000 === 0) + console.log(`Processing record ${x} of ${records.length}`); const record = records[x]; const pluginCharacteristics = {} as { [key: string]: string }; for (const pluginCharacteristicsAttribute of PLUGIN_CHARACTERISTICS_ATTRIBUTES) { if (pluginCharacteristicsAttribute in record) { if (record[pluginCharacteristicsAttribute] !== '') { - pluginCharacteristics[pluginCharacteristicsAttribute] = record[pluginCharacteristicsAttribute]; + pluginCharacteristics[pluginCharacteristicsAttribute] = + record[pluginCharacteristicsAttribute]; } delete record[pluginCharacteristicsAttribute]; } } - record.pluginsData = Object.keys(pluginCharacteristics).length !== 0 ? pluginCharacteristics : missingValueDatasetToken; + record.pluginsData = + Object.keys(pluginCharacteristics).length !== 0 + ? pluginCharacteristics + : missingValueDatasetToken; for (const attribute of Object.keys(record)) { if ([null, '', undefined].includes(record[attribute])) { record[attribute] = missingValueDatasetToken; } else { - record[attribute] = (typeof record[attribute] === 'string' || record[attribute] instanceof String) - ? record[attribute] - : (STRINGIFIED_PREFIX + JSON.stringify(record[attribute])); + record[attribute] = + typeof record[attribute] === 'string' || + record[attribute] instanceof String + ? record[attribute] + : STRINGIFIED_PREFIX + + JSON.stringify(record[attribute]); } } records[x] = record; } - const fingerprintGeneratorNetwork = new BayesianNetwork({ path: path.join(__dirname, 'network_structures', 'fingerprint-network-structure.zip') }); + const fingerprintGeneratorNetwork = new BayesianNetwork({ + path: path.join( + __dirname, + 'network_structures', + 'fingerprint-network-structure.zip' + ), + }); // eslint-disable-next-line dot-notation - const desiredFingerprintAttributes = Object.keys(fingerprintGeneratorNetwork['nodesByName']); + const desiredFingerprintAttributes = Object.keys( + fingerprintGeneratorNetwork['nodesByName'] + ); const selectedRecords = records.map((record) => { - return Object.entries(record).reduce((acc: typeof record, [key, value]) => { - if (desiredFingerprintAttributes.includes(key)) acc[key] = value ?? missingValueDatasetToken; - return acc; - }, {}); + return Object.entries(record).reduce( + (acc: typeof record, [key, value]) => { + if (desiredFingerprintAttributes.includes(key)) + acc[key] = value ?? missingValueDatasetToken; + return acc; + }, + {} + ); }); - const fingerprintNetworkDefinitionPath = path.join(resultsPath, 'fingerprint-network-definition.zip'); + const fingerprintNetworkDefinitionPath = path.join( + resultsPath, + 'fingerprint-network-definition.zip' + ); // eslint-disable-next-line no-console console.log('Building the fingerprint network...'); - fingerprintGeneratorNetwork.setProbabilitiesAccordingToData(selectedRecords); - fingerprintGeneratorNetwork.saveNetworkDefinition({ path: fingerprintNetworkDefinitionPath }); + fingerprintGeneratorNetwork.setProbabilitiesAccordingToData( + selectedRecords + ); + fingerprintGeneratorNetwork.saveNetworkDefinition({ + path: fingerprintNetworkDefinitionPath, + }); } } From 5921fd882509b300c604b94e85027b139b39ab3a Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 16:56:17 -0500 Subject: [PATCH 03/11] feat: Add validation for productSub --- .../src/generator-networks-creator.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index b54e9764..c9cf80a2 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -112,6 +112,13 @@ async function prepareRecords( // The maxTouchPoints should be 0 for desktops and > 0 for mobile devices if (!validTouchSupport) continue; + const validProductSub = + parsedUserAgent.browser.name === 'Firefox' || + fingerprint.productSub === '20030107'; + + // The productSub should be 20030107 for Non-Firefox supported browsers + if (!validProductSub) continue; + cleanedRecords.push({ ...record, userAgent: record.browserFingerprint.userAgent, From f585f1ecba19470e0eae029ffc409686875f225f Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 17:02:24 -0500 Subject: [PATCH 04/11] feat: Enhance user agent validation for productSub and vendor checks --- .../src/generator-networks-creator.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index c9cf80a2..7359552e 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -79,7 +79,7 @@ async function prepareRecords( ).withClientHints(); const validBrowser = - parsedUserAgent.browser.name && + parsedUserAgent.browser.name !== undefined && [ 'Edge', 'Chrome', @@ -113,12 +113,23 @@ async function prepareRecords( if (!validTouchSupport) continue; const validProductSub = - parsedUserAgent.browser.name === 'Firefox' || - fingerprint.productSub === '20030107'; + parsedUserAgent.browser.name === 'Firefox' + ? fingerprint.productSub === '20100101' + : fingerprint.productSub === '20030107'; // The productSub should be 20030107 for Non-Firefox supported browsers if (!validProductSub) continue; + const validVendor = + (parsedUserAgent.browser.name === 'Firefox' && + fingerprint.vendor === '') || + (parsedUserAgent.browser.name!.startsWith('Safari') && + fingerprint.vendor === 'Apple Computer, Inc.') || + fingerprint.vendor === 'Google Inc.'; + + // The vendor should be Google Inc. for Chrome and Apple Computer, Inc. for Safari + if (!validVendor) continue; + cleanedRecords.push({ ...record, userAgent: record.browserFingerprint.userAgent, From 6f697bffa449e5a82166371dbf0b1f92653f499c Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 17:07:06 -0500 Subject: [PATCH 05/11] feat: Update user agent validation for product and app name checks --- .../src/generator-networks-creator.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index 7359552e..0093c42e 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -112,13 +112,14 @@ async function prepareRecords( // The maxTouchPoints should be 0 for desktops and > 0 for mobile devices if (!validTouchSupport) continue; - const validProductSub = + const validProduct = + fingerprint.product === 'Gecko' && parsedUserAgent.browser.name === 'Firefox' ? fingerprint.productSub === '20100101' : fingerprint.productSub === '20030107'; - // The productSub should be 20030107 for Non-Firefox supported browsers - if (!validProductSub) continue; + // The productSub should be 20100101 for Firefox and 20030107 for the rest + if (!validProduct) continue; const validVendor = (parsedUserAgent.browser.name === 'Firefox' && @@ -130,6 +131,13 @@ async function prepareRecords( // The vendor should be Google Inc. for Chrome and Apple Computer, Inc. for Safari if (!validVendor) continue; + const validAppName = + fingerprint.appName === 'Netscape' && + fingerprint.appCodeName === 'Mozilla'; + + // The appName should be Netscape and the appCodeName should be Mozilla + if (!validAppName) continue; + cleanedRecords.push({ ...record, userAgent: record.browserFingerprint.userAgent, From 8df389b30114ab8ec4d8218fcfa8395778cf16aa Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 17:30:58 -0500 Subject: [PATCH 06/11] feat: Add validation for known WebGL renderer and improve renderer checks --- .../src/generator-networks-creator.ts | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index 0093c42e..dc6efc7a 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -23,6 +23,76 @@ const STRINGIFIED_PREFIX = '*STRINGIFIED*'; const PLUGIN_CHARACTERISTICS_ATTRIBUTES = ['plugins', 'mimeTypes']; +const BLANK_SPACE_NOISE_REGEX = /\s{2,}|^\s|\s$/; + +const ANGLE_WEBGL_RENDERER_REGEX = /^ANGLE/; + +const ANGLE_BRACKETS_WEBGL_RENDERER_REGEX = /^ANGLE \((.+)\)/; + +const KNOWN_WEBGL_RENDERER_PARTS = [ + 'AMD', + 'ANGLE', + 'ASUS', + 'ATI', + 'ATI Radeon', + 'ATI Technologies Inc', + 'Adreno', + 'Android Emulator', + 'Apple', + 'Apple GPU', + 'Apple M1', + 'Chipset', + 'D3D11', + 'Direct3D', + 'Express Chipset', + 'GeForce', + 'Generation', + 'Generic Renderer', + 'Google', + 'Google SwiftShader', + 'Graphics', + 'Graphics Media Accelerator', + 'HD Graphics Family', + 'Intel', + 'Intel(R) HD Graphics', + 'Intel(R) UHD Graphics', + 'Iris', + 'KBL Graphics', + 'Mali', + 'Mesa', + 'Mesa DRI', + 'Metal', + 'Microsoft', + 'Microsoft Basic Render Driver', + 'Microsoft Corporation', + 'NVIDIA', + 'NVIDIA Corporation', + 'NVIDIAGameReadyD3D', + 'OpenGL', + 'OpenGL Engine', + 'Open Source Technology Center', + 'Parallels', + 'Parallels Display Adapter', + 'PCIe', + 'Plus Graphics', + 'PowerVR', + 'Pro Graphics', + 'Quadro', + 'Radeon', + 'Radeon Pro', + 'Radeon Pro Vega', + 'Samsung', + 'SSE2', + 'VMware', + 'VMware SVGA 3D', + 'Vega', + 'VirtualBox', + 'VirtualBox Graphics Adapter', + 'Vulkan', + 'Xe Graphics', + 'llvmpipe', +]; + async function prepareRecords( records: Record[], preprocessingType: string @@ -138,6 +208,23 @@ async function prepareRecords( // The appName should be Netscape and the appCodeName should be Mozilla if (!validAppName) continue; + const knownWebGLRenderer = + !!fingerprint.videoCard && + KNOWN_WEBGL_RENDERER_PARTS.some((name) => + fingerprint.videoCard.renderer.includes(name) + ); + + const validWebGLRenderer = + knownWebGLRenderer && + !BLANK_SPACE_NOISE_REGEX.test(fingerprint.videoCard.renderer) && + (!ANGLE_WEBGL_RENDERER_REGEX.test(fingerprint.videoCard.renderer) || + !!(ANGLE_BRACKETS_WEBGL_RENDERER_REGEX.exec( + fingerprint.videoCard.renderer + ) || [])[1]); + + // The WebGL renderer should contain a known substring, should not contain excessive blank spaces, should not be ANGLE or should be ANGLE with a valid name + if (!validWebGLRenderer) continue; + cleanedRecords.push({ ...record, userAgent: record.browserFingerprint.userAgent, From fba50bdf3cd301ca921f4edf0bcaa89d2f32b6fc Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 18:34:31 -0500 Subject: [PATCH 07/11] feat: Add validation for known OS fonts based on the user's operating system --- .../src/generator-networks-creator.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index dc6efc7a..24c6ae9b 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -93,6 +93,32 @@ const KNOWN_WEBGL_RENDERER_PARTS = [ 'llvmpipe', ]; +const KNOWN_OS_FONTS = { + WINDOWS: [ + 'Cambria Math', + 'Nirmala UI', + 'Leelawadee UI', + 'HoloLens MDL2 Assets', + 'Segoe Fluent Icons', + ], + APPLE: [ + 'Helvetica Neue', + 'Luminari', + 'PingFang HK Light', + 'InaiMathi Bold', + 'Galvji', + 'Chakra Petch', + ], + LINUX: [ + 'Arimo', + 'MONO', + 'Ubuntu', + 'Noto Color Emoji', + 'Dancing Script', + 'Droid Sans Mono', + ], +}; + async function prepareRecords( records: Record[], preprocessingType: string @@ -225,6 +251,29 @@ async function prepareRecords( // The WebGL renderer should contain a known substring, should not contain excessive blank spaces, should not be ANGLE or should be ANGLE with a valid name if (!validWebGLRenderer) continue; + const fontValidatableOS = + parsedUserAgent.os.name && + (parsedUserAgent.os.name.startsWith('Windows') || + ['macOS', 'iOS'].includes(parsedUserAgent.os.name)); + + const validFonts = + fingerprint.fonts.length === 0 || + !fontValidatableOS || + fingerprint.fonts.some((font: string) => { + if (parsedUserAgent.os.name!.startsWith('Windows')) { + return KNOWN_OS_FONTS.WINDOWS.includes(font); + } else if ( + ['macOS', 'iOS'].includes(parsedUserAgent.os.name!) + ) { + return KNOWN_OS_FONTS.APPLE.includes(font); + } + + return false; + }); + + // The fonts should be empty or contain only known fonts for the OS + if (!validFonts) continue; + cleanedRecords.push({ ...record, userAgent: record.browserFingerprint.userAgent, From c74ea989a14bfbe7dc320769f6ab2daa92969650 Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 18:44:52 -0500 Subject: [PATCH 08/11] feat: Add validation for screen dimensions and device pixel ratio in user agent fingerprinting --- .../src/generator-networks-creator.ts | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index 24c6ae9b..28fb23eb 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -274,6 +274,50 @@ async function prepareRecords( // The fonts should be empty or contain only known fonts for the OS if (!validFonts) continue; + const validScreenSize = + fingerprint.screen.width > 0 && + fingerprint.screen.height > 0 && + fingerprint.screen.availWidth > 0 && + fingerprint.screen.availHeight > 0; + + // The screen dimensions should be positive + if (!validScreenSize) continue; + + const validAvailDimensions = + fingerprint.screen.availWidth <= fingerprint.screen.width && + fingerprint.screen.availHeight <= fingerprint.screen.height; + + // The availWidth and availHeight should be less or equal to the width and height + if (!validAvailDimensions) continue; + + const validWindowSize = + fingerprint.screen.innerWidth <= fingerprint.screen.outerWidth && + fingerprint.screen.innerHeight <= fingerprint.screen.outerHeight; + + // The innerWidth and innerHeight should be less or equal to the outerWidth and outerHeight + if (!validWindowSize) continue; + + const validClientDimensions = + fingerprint.screen.clientWidth <= fingerprint.screen.innerWidth && + fingerprint.screen.clientHeight <= fingerprint.screen.innerHeight; + + // The clientWidth and clientHeight should be less or equal to the innerWidth and innerHeight + if (!validClientDimensions) continue; + + const validColorDepth = + !fingerprint.screen.pixelDepth || + fingerprint.screen.pixelDepth === fingerprint.screen.colorDepth; + + // The pixelDepth and colorDepth should be equal + if (!validColorDepth) continue; + + const validDevicePixelRatio = + fingerprint.screen.devicePixelRatio >= 1 && + fingerprint.screen.screen.devicePixelRatio <= 5; + + // The devicePixelRatio should be between 1 and 5 + if (!validDevicePixelRatio) continue; + cleanedRecords.push({ ...record, userAgent: record.browserFingerprint.userAgent, From b189b649a72641d426385f1744e7c018bdcb488b Mon Sep 17 00:00:00 2001 From: ARYA Date: Thu, 23 Jan 2025 18:57:33 -0500 Subject: [PATCH 09/11] feat: Add validation for screen dimensions and aspect ratios in user agent fingerprinting --- .../src/generator-networks-creator.ts | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index 28fb23eb..020476ac 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -313,11 +313,50 @@ async function prepareRecords( const validDevicePixelRatio = fingerprint.screen.devicePixelRatio >= 1 && - fingerprint.screen.screen.devicePixelRatio <= 5; + fingerprint.screen.devicePixelRatio <= 5; // The devicePixelRatio should be between 1 and 5 if (!validDevicePixelRatio) continue; + let validScreenDimensions = true; + + if (desktopFingerprint) { + const aspectRatio = + fingerprint.screen.width / fingerprint.screen.height; + + validScreenDimensions = + fingerprint.screen.width >= 1024 && + fingerprint.screen.width <= 5120 && + fingerprint.screen.height >= 768 && + fingerprint.screen.height <= 2880 && + (Math.abs(aspectRatio - 16 / 9) < 0.1 || // 16:9 aspect ratio + Math.abs(aspectRatio - 4 / 3) < 0.1); // 4:3 aspect ratio + } else { + const screenWidth = Math.max( + fingerprint.screen.width, + fingerprint.screen.height + ); + + const screenHeight = Math.min( + fingerprint.screen.width, + fingerprint.screen.height + ); + + const screenAspectRatio = screenWidth / screenHeight; + + validScreenDimensions = + screenWidth >= 320 && + screenWidth <= 2560 && + screenHeight >= 480 && + screenHeight <= 3200 && + (Math.abs(screenAspectRatio - 9 / 16) < 0.1 || // 9:16 aspect ratio + Math.abs(screenAspectRatio - 4 / 3) < 0.1 || // 4:3 aspect ratio + Math.abs(screenAspectRatio - 16 / 9) < 0.1); // 16:9 aspect ratio + } + + // The screen dimensions should be within the expected range + if (!validScreenDimensions) continue; + cleanedRecords.push({ ...record, userAgent: record.browserFingerprint.userAgent, From 3d786367fc793d8273eaf94a1bef6d2733e08421 Mon Sep 17 00:00:00 2001 From: ARYA Date: Fri, 24 Jan 2025 05:12:33 -0500 Subject: [PATCH 10/11] feat: Simplify screen dimension validation by removing aspect ratio checks --- .../src/generator-networks-creator.ts | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index 020476ac..8322391b 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -321,16 +321,11 @@ async function prepareRecords( let validScreenDimensions = true; if (desktopFingerprint) { - const aspectRatio = - fingerprint.screen.width / fingerprint.screen.height; - validScreenDimensions = fingerprint.screen.width >= 1024 && fingerprint.screen.width <= 5120 && fingerprint.screen.height >= 768 && - fingerprint.screen.height <= 2880 && - (Math.abs(aspectRatio - 16 / 9) < 0.1 || // 16:9 aspect ratio - Math.abs(aspectRatio - 4 / 3) < 0.1); // 4:3 aspect ratio + fingerprint.screen.height <= 2880; } else { const screenWidth = Math.max( fingerprint.screen.width, @@ -342,16 +337,11 @@ async function prepareRecords( fingerprint.screen.height ); - const screenAspectRatio = screenWidth / screenHeight; - validScreenDimensions = screenWidth >= 320 && screenWidth <= 2560 && screenHeight >= 480 && - screenHeight <= 3200 && - (Math.abs(screenAspectRatio - 9 / 16) < 0.1 || // 9:16 aspect ratio - Math.abs(screenAspectRatio - 4 / 3) < 0.1 || // 4:3 aspect ratio - Math.abs(screenAspectRatio - 16 / 9) < 0.1); // 16:9 aspect ratio + screenHeight <= 3200; } // The screen dimensions should be within the expected range From f14dac7efc98c590089146c7511c3f5756ddef10 Mon Sep 17 00:00:00 2001 From: ARYA Date: Fri, 24 Jan 2025 06:10:31 -0500 Subject: [PATCH 11/11] feat: Update screen dimension validation for desktop fingerprints to allow smaller resolutions --- .../src/generator-networks-creator.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/generator-networks-creator/src/generator-networks-creator.ts b/packages/generator-networks-creator/src/generator-networks-creator.ts index 8322391b..92b0a205 100755 --- a/packages/generator-networks-creator/src/generator-networks-creator.ts +++ b/packages/generator-networks-creator/src/generator-networks-creator.ts @@ -322,10 +322,8 @@ async function prepareRecords( if (desktopFingerprint) { validScreenDimensions = - fingerprint.screen.width >= 1024 && - fingerprint.screen.width <= 5120 && - fingerprint.screen.height >= 768 && - fingerprint.screen.height <= 2880; + fingerprint.screen.width >= 512 && + fingerprint.screen.height >= 384; } else { const screenWidth = Math.max( fingerprint.screen.width,