update useragent logic

cockroacher · May 19, 2024 · 31a6b47 · 31a6b47
1 parent db8be16
commit 31a6b47
Show file tree

Hide file tree

Showing 5 changed files with 82 additions and 26 deletions.
diff --git a/.github/workflows/update-software.yml b/.github/workflows/update-software.yml
@@ -43,9 +43,7 @@ jobs:
         google-chrome --version
     - name: Setup npm packages
       run: npm install
-    - name: Setup config
-      run: python tools/verify_result.py -c ${{ matrix.config }},GITHUB_API_KEY=${{ secrets.GITHUB_TOKEN }}
-    - name: Update USER_AGENT in our defaults/config.py
+    - name: Update USER_AGENT in our defaults/settings.json
       run: python update_software.py -b
     - name: Checkout advisory-database repo
       uses: actions/checkout@v4
@@ -71,11 +69,11 @@ jobs:
             - software-sources.json
             - software-full.json
             - defaults/software-rules.json
-            - defaults/config.py
+            - defaults/settings.json
         assignees: 7h3Rabbit
         reviewers: 7h3Rabbit
         add-paths: |
           software-sources.json
           software-full.json
           defaults/software-rules.json
-          defaults/config.py
+          defaults/settings.json
diff --git a/defaults/settings.json b/defaults/settings.json
@@ -10,7 +10,7 @@
             "details": false,
             "improve-only": true
         },
-        "useragent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",
+        "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
         "cache": {
             "use": false,
             "max-age": 60

diff --git a/helpers/setting_helper.py b/helpers/setting_helper.py
@@ -141,7 +141,22 @@ def get_config(name):
 
     return None
 
-def set_config(name, value):
+def update_config(name, value, module_name):
+    if '.' not in name:
+        # Try translate old settings name to new
+        config_name = get_setting_name(name)
+        if config_name is None:
+            print(f'Warning: {name} uses old settings format and is not a known setting')
+            return
+
+        config_name_pair = config_name.split('|')
+        name = config_name_pair[1]
+
+    name = name.lower()
+    # Try set config for our configuration file
+    update_config_for_module(name, value, module_name)
+
+def set_runtime_config_only(name, value):
     """
     Set a configuration value.
 
@@ -152,6 +167,31 @@ def set_config(name, value):
     name = name.lower()
     config[name] = value
 
+def update_config_for_module(config_name, config_value, module_name):
+    base_directory = Path(os.path.dirname(
+        os.path.realpath(__file__)) + os.path.sep).parent
+
+    file_path = f'{base_directory}{os.path.sep}{module_name}'
+    if not os.path.isfile(file_path):
+        return
+
+    module_config = {}
+    with open(file_path, encoding='utf-8') as json_file:
+        module_config = json.load(json_file)
+
+        config_path = config_name.split('.')
+        config_section = module_config
+        for section_name in config_path:
+            if section_name in config_section:
+                tmp_config_section = config_section[section_name]
+                if not isinstance(tmp_config_section, dict):
+                    config_section[section_name] = config_value
+                    break
+                config_section = tmp_config_section
+
+    with open(file_path, 'w', encoding='utf-8') as outfile:
+        json.dump(module_config, outfile)
+
 def get_config_from_module(config_name, module_name):
     """
     Retrieves the configuration value for a given name from the specified module file.
@@ -222,7 +262,7 @@ def set_config_from_cmd(arg):
     for value_type, handler in value_type_handlers.items():
         if config_name_type in value_type:
             config_value = handler(config_name, value)
-            set_config(config_name.lower(), config_value)
+            set_runtime_config_only(config_name.lower(), config_value)
             return True
     return False
 

diff --git a/tools/verify_result.py b/tools/verify_result.py
@@ -456,8 +456,6 @@ def main(argv):
 
     Options and arguments:
     -h/--help\t\t\t: Verify Help command
-    -c/--prep-config <activate feature, True or False>\t\t:
-      Uses defaults/config.py to create config.py
     -d/--docker <activate feature, True or False>\t\t:
       Updates DockerFile to use latest browsers
     -t/--test <test number>\t: Verify result of specific test
@@ -469,9 +467,9 @@ def main(argv):
     """
 
     try:
-        opts, _ = getopt.getopt(argv, "hlc:t:d:s:", [
-                                   "help", "test=", "prep-config=",
-                                   "sample-config=", "language", "docker="])
+        opts, _ = getopt.getopt(argv, "hlt:d:s:", [
+                                   "help", "test=", "sample-config=",
+                                   "language", "docker="])
     except getopt.GetoptError:
         print(main.__doc__)
         sys.exit(2)
@@ -484,8 +482,6 @@ def main(argv):
         if opt in ('-h', '--help'):  # help
             print(main.__doc__)
             sys.exit(0)
-        elif opt in ("-c", "--prep-config"):
-            handle_pre_config(arg)
         elif opt in ("-s", "--sample-config"):
             handle_sample_config(arg)
         elif opt in ("-t", "--test"):  # test id

diff --git a/update_software.py b/update_software.py
@@ -13,8 +13,7 @@
 from tests.sitespeed_base import get_browsertime_har_path,\
     get_result_using_no_cache, get_sanitized_browsertime
 from tests.utils import get_http_content
-from tools.verify_result import handle_sample_config
-from helpers.setting_helper import get_config
+from helpers.setting_helper import get_config, update_config
 
 USE_CACHE = get_config('general.cache.use')
 CACHE_TIME_DELTA = timedelta(minutes=get_config('general.cache.max-age'))
@@ -65,18 +64,41 @@ def update_user_agent():
     result_folder_name = os.path.join(folder, hostname, f'{str(uuid.uuid4())}')
 
     sitespeed_iterations = 1
-    sitespeed_arg = '--plugins.remove screenshot --plugins.remove html --plugins.remove metrics --browsertime.screenshot false --screenshot false --screenshotLCP false --browsertime.screenshotLCP false --videoParams.createFilmstrip false --visualMetrics false --visualMetricsPerceptual false --visualMetricsContentful false --browsertime.headless true --utc true -n {0}'.format(
-        sitespeed_iterations)
+    sitespeed_arg = (
+        '--plugins.remove screenshot '
+        '--plugins.remove html '
+        '--plugins.remove metrics '
+        '--browsertime.screenshot false '
+        '--screenshot false '
+        '--screenshotLCP false '
+        '--browsertime.screenshotLCP false '
+        '--videoParams.createFilmstrip false '
+        '--visualMetrics false '
+        '--visualMetricsPerceptual false '
+        '--visualMetricsContentful false '
+        '--browsertime.headless true '
+        '--utc true '
+        f'-n {sitespeed_iterations}')
 
     if 'firefox' in software_browser:
-        sitespeed_arg = '-b firefox --firefox.includeResponseBodies all --firefox.preference privacy.trackingprotection.enabled:false --firefox.preference privacy.donottrackheader.enabled:false --firefox.preference browser.safebrowsing.malware.enabled:false --firefox.preference browser.safebrowsing.phishing.enabled:false {0}'.format(
-            sitespeed_arg)
+        sitespeed_arg = (
+            '-b firefox '
+            '--firefox.includeResponseBodies all '
+            '--firefox.preference privacy.trackingprotection.enabled:false '
+            '--firefox.preference privacy.donottrackheader.enabled:false '
+            '--firefox.preference browser.safebrowsing.malware.enabled:false '
+            '--firefox.preference browser.safebrowsing.phishing.enabled:false '
+            f'{sitespeed_arg}')
     else:
-        sitespeed_arg = '-b chrome --chrome.cdp.performance false --browsertime.chrome.timeline false --browsertime.chrome.includeResponseBodies all --browsertime.chrome.args ignore-certificate-errors {0}'.format(
-            sitespeed_arg)
+        sitespeed_arg = (
+            '-b chrome '
+            '--chrome.cdp.performance false '
+            '--browsertime.chrome.timeline false '
+            '--browsertime.chrome.includeResponseBodies all '
+            '--browsertime.chrome.args ignore-certificate-errors '
+            f'{sitespeed_arg}')
 
-    sitespeed_arg = '--shm-size=1g {0}'.format(
-        sitespeed_arg)
+    sitespeed_arg = f'--shm-size=1g {sitespeed_arg}'
 
     if 'nt' not in os.name:
         sitespeed_arg += ' --xvfb'
@@ -114,7 +136,7 @@ def update_user_agent():
                 continue
 
             print(header_value)
-            handle_sample_config(f"USERAGENT={header_value}")
+            update_config('general.useragent', header_value, f'defaults{os.path.sep}settings.json')
             return