From 31a6b47f25f03b79acc36deaec878d87e16bc4e8 Mon Sep 17 00:00:00 2001 From: cockroacher <163405488+cockroacher@users.noreply.github.com> Date: Sun, 19 May 2024 13:35:07 +0200 Subject: [PATCH] update useragent logic --- .github/workflows/update-software.yml | 8 ++--- defaults/settings.json | 2 +- helpers/setting_helper.py | 44 +++++++++++++++++++++++++-- tools/verify_result.py | 10 ++---- update_software.py | 44 ++++++++++++++++++++------- 5 files changed, 82 insertions(+), 26 deletions(-) diff --git a/.github/workflows/update-software.yml b/.github/workflows/update-software.yml index 13100ae8..362c6741 100644 --- a/.github/workflows/update-software.yml +++ b/.github/workflows/update-software.yml @@ -43,9 +43,7 @@ jobs: google-chrome --version - name: Setup npm packages run: npm install - - name: Setup config - run: python tools/verify_result.py -c ${{ matrix.config }},GITHUB_API_KEY=${{ secrets.GITHUB_TOKEN }} - - name: Update USER_AGENT in our defaults/config.py + - name: Update USER_AGENT in our defaults/settings.json run: python update_software.py -b - name: Checkout advisory-database repo uses: actions/checkout@v4 @@ -71,11 +69,11 @@ jobs: - software-sources.json - software-full.json - defaults/software-rules.json - - defaults/config.py + - defaults/settings.json assignees: 7h3Rabbit reviewers: 7h3Rabbit add-paths: | software-sources.json software-full.json defaults/software-rules.json - defaults/config.py + defaults/settings.json diff --git a/defaults/settings.json b/defaults/settings.json index 64695d62..ed4f5a01 100644 --- a/defaults/settings.json +++ b/defaults/settings.json @@ -10,7 +10,7 @@ "details": false, "improve-only": true }, - "useragent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0", + "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0", "cache": { "use": false, "max-age": 60 diff --git a/helpers/setting_helper.py b/helpers/setting_helper.py index c8968a78..bceff8fb 100644 --- a/helpers/setting_helper.py +++ b/helpers/setting_helper.py @@ -141,7 +141,22 @@ def get_config(name): return None -def set_config(name, value): +def update_config(name, value, module_name): + if '.' not in name: + # Try translate old settings name to new + config_name = get_setting_name(name) + if config_name is None: + print(f'Warning: {name} uses old settings format and is not a known setting') + return + + config_name_pair = config_name.split('|') + name = config_name_pair[1] + + name = name.lower() + # Try set config for our configuration file + update_config_for_module(name, value, module_name) + +def set_runtime_config_only(name, value): """ Set a configuration value. @@ -152,6 +167,31 @@ def set_config(name, value): name = name.lower() config[name] = value +def update_config_for_module(config_name, config_value, module_name): + base_directory = Path(os.path.dirname( + os.path.realpath(__file__)) + os.path.sep).parent + + file_path = f'{base_directory}{os.path.sep}{module_name}' + if not os.path.isfile(file_path): + return + + module_config = {} + with open(file_path, encoding='utf-8') as json_file: + module_config = json.load(json_file) + + config_path = config_name.split('.') + config_section = module_config + for section_name in config_path: + if section_name in config_section: + tmp_config_section = config_section[section_name] + if not isinstance(tmp_config_section, dict): + config_section[section_name] = config_value + break + config_section = tmp_config_section + + with open(file_path, 'w', encoding='utf-8') as outfile: + json.dump(module_config, outfile) + def get_config_from_module(config_name, module_name): """ Retrieves the configuration value for a given name from the specified module file. @@ -222,7 +262,7 @@ def set_config_from_cmd(arg): for value_type, handler in value_type_handlers.items(): if config_name_type in value_type: config_value = handler(config_name, value) - set_config(config_name.lower(), config_value) + set_runtime_config_only(config_name.lower(), config_value) return True return False diff --git a/tools/verify_result.py b/tools/verify_result.py index 3d0b7b6f..2c59cb43 100644 --- a/tools/verify_result.py +++ b/tools/verify_result.py @@ -456,8 +456,6 @@ def main(argv): Options and arguments: -h/--help\t\t\t: Verify Help command - -c/--prep-config \t\t: - Uses defaults/config.py to create config.py -d/--docker \t\t: Updates DockerFile to use latest browsers -t/--test \t: Verify result of specific test @@ -469,9 +467,9 @@ def main(argv): """ try: - opts, _ = getopt.getopt(argv, "hlc:t:d:s:", [ - "help", "test=", "prep-config=", - "sample-config=", "language", "docker="]) + opts, _ = getopt.getopt(argv, "hlt:d:s:", [ + "help", "test=", "sample-config=", + "language", "docker="]) except getopt.GetoptError: print(main.__doc__) sys.exit(2) @@ -484,8 +482,6 @@ def main(argv): if opt in ('-h', '--help'): # help print(main.__doc__) sys.exit(0) - elif opt in ("-c", "--prep-config"): - handle_pre_config(arg) elif opt in ("-s", "--sample-config"): handle_sample_config(arg) elif opt in ("-t", "--test"): # test id diff --git a/update_software.py b/update_software.py index 4e91eaaf..2e0605c1 100644 --- a/update_software.py +++ b/update_software.py @@ -13,8 +13,7 @@ from tests.sitespeed_base import get_browsertime_har_path,\ get_result_using_no_cache, get_sanitized_browsertime from tests.utils import get_http_content -from tools.verify_result import handle_sample_config -from helpers.setting_helper import get_config +from helpers.setting_helper import get_config, update_config USE_CACHE = get_config('general.cache.use') CACHE_TIME_DELTA = timedelta(minutes=get_config('general.cache.max-age')) @@ -65,18 +64,41 @@ def update_user_agent(): result_folder_name = os.path.join(folder, hostname, f'{str(uuid.uuid4())}') sitespeed_iterations = 1 - sitespeed_arg = '--plugins.remove screenshot --plugins.remove html --plugins.remove metrics --browsertime.screenshot false --screenshot false --screenshotLCP false --browsertime.screenshotLCP false --videoParams.createFilmstrip false --visualMetrics false --visualMetricsPerceptual false --visualMetricsContentful false --browsertime.headless true --utc true -n {0}'.format( - sitespeed_iterations) + sitespeed_arg = ( + '--plugins.remove screenshot ' + '--plugins.remove html ' + '--plugins.remove metrics ' + '--browsertime.screenshot false ' + '--screenshot false ' + '--screenshotLCP false ' + '--browsertime.screenshotLCP false ' + '--videoParams.createFilmstrip false ' + '--visualMetrics false ' + '--visualMetricsPerceptual false ' + '--visualMetricsContentful false ' + '--browsertime.headless true ' + '--utc true ' + f'-n {sitespeed_iterations}') if 'firefox' in software_browser: - sitespeed_arg = '-b firefox --firefox.includeResponseBodies all --firefox.preference privacy.trackingprotection.enabled:false --firefox.preference privacy.donottrackheader.enabled:false --firefox.preference browser.safebrowsing.malware.enabled:false --firefox.preference browser.safebrowsing.phishing.enabled:false {0}'.format( - sitespeed_arg) + sitespeed_arg = ( + '-b firefox ' + '--firefox.includeResponseBodies all ' + '--firefox.preference privacy.trackingprotection.enabled:false ' + '--firefox.preference privacy.donottrackheader.enabled:false ' + '--firefox.preference browser.safebrowsing.malware.enabled:false ' + '--firefox.preference browser.safebrowsing.phishing.enabled:false ' + f'{sitespeed_arg}') else: - sitespeed_arg = '-b chrome --chrome.cdp.performance false --browsertime.chrome.timeline false --browsertime.chrome.includeResponseBodies all --browsertime.chrome.args ignore-certificate-errors {0}'.format( - sitespeed_arg) + sitespeed_arg = ( + '-b chrome ' + '--chrome.cdp.performance false ' + '--browsertime.chrome.timeline false ' + '--browsertime.chrome.includeResponseBodies all ' + '--browsertime.chrome.args ignore-certificate-errors ' + f'{sitespeed_arg}') - sitespeed_arg = '--shm-size=1g {0}'.format( - sitespeed_arg) + sitespeed_arg = f'--shm-size=1g {sitespeed_arg}' if 'nt' not in os.name: sitespeed_arg += ' --xvfb' @@ -114,7 +136,7 @@ def update_user_agent(): continue print(header_value) - handle_sample_config(f"USERAGENT={header_value}") + update_config('general.useragent', header_value, f'defaults{os.path.sep}settings.json') return