pylint

Author: cockroacher

engines/sitemap.py

@@ -5,7 +5,7 @@
 import io
 from bs4 import BeautifulSoup
 from engines.utils import use_item
-from tests.utils import get_content_type, httpRequestGetContent, merge_dicts, cache_time_delta
+from tests.utils import get_content_type, get_http_content, merge_dicts, CACHE_TIME_DELTA
 
 def read_sites(input_sitemap_url, input_skip, input_take):
     ignore_none_html = True
@@ -25,7 +25,7 @@ def read_sitemap(input_sitemap_url, input_skip, input_take, ignore_none_html):
 
     if input_sitemap_url.endswith('.xml.gz'):
         # unpack gzip:ed sitemap
-        sitemap_content = httpRequestGetContent(input_sitemap_url, True, False)
+        sitemap_content = get_http_content(input_sitemap_url, True, False)
         try:
             if isinstance(sitemap_content, str):
                 return result
@@ -42,7 +42,7 @@ def read_sitemap(input_sitemap_url, input_skip, input_take, ignore_none_html):
         except gzip.BadGzipFile:
             return result
     else:
-        sitemap_content = httpRequestGetContent(input_sitemap_url, True, True)
+        sitemap_content = get_http_content(input_sitemap_url, True, True)
         result = merge_dicts(read_sitemap_xml(input_sitemap_url,
             sitemap_content,
             input_skip,
@@ -113,7 +113,7 @@ def read_sitemap_xml(key, sitemap_content, input_skip, input_take, ignore_none_h
                     print(f'- skipping because it is of type: {item_type}')
                     continue
 
-                item_content_type = get_content_type(item_url, cache_time_delta)
+                item_content_type = get_content_type(item_url, CACHE_TIME_DELTA)
                 print('content-type', item_content_type)
                 if item_content_type == 401:
                     print(f'- skipping because it is of status-code: {item_content_type}')

engines/webperf.py

@@ -24,7 +24,7 @@ def read_sites(input_url, input_skip, input_take):
     else:
         raise NotImplementedError('input is incorrect')
 
-    category_content = httpRequestGetContent(input_url)
+    category_content = get_http_content(input_url)
 
     category_regex = r"<a href=\"(?P<detail_url>\/site\/[^\"]+)\""
     category_matches = re.finditer(
@@ -43,7 +43,7 @@ def read_sites(input_url, input_skip, input_take):
     detail_regex = r"Webbplats:<\/th>[ \r\n\t]+<td><a href=\"(?P<item_url>[^\"]+)\""
     current_index = 0
     for detail_url in detailed_urls:
-        detail_content = httpRequestGetContent(detail_url)
+        detail_content = get_http_content(detail_url)
         detail_match = re.search(detail_regex, detail_content, re.MULTILINE)
         item_url = detail_match.group('item_url')
 

test_dnssec.py

@@ -243,7 +243,7 @@ def validate_dnssec(domain, domain_entry):
 def testdns(key, datatype, use_dnssec):
     print('\ttestdns', key, datatype, use_dnssec)
     cache_key = 'dnslookup://{0}#{1}#{2}'.format(key, datatype, use_dnssec)
-    if has_cache_file(cache_key, True, cache_time_delta):
+    if has_cache_file(cache_key, True, CACHE_TIME_DELTA):
         cache_path = get_cache_path(cache_key, True)
         print('\t- Using dnslookup cache')
         response = dns.message.from_file(cache_path)

tests/a11y_statement.py

@@ -8,7 +8,7 @@
 from bs4 import BeautifulSoup
 import gettext
 
-from tests.utils import httpRequestGetContent
+from tests.utils import get_http_content
 _ = gettext.gettext
 
 review_show_improvements_only = config.review_show_improvements_only
@@ -64,7 +64,7 @@ def run_test(_, langCode, url):
 
 
 def get_digg_report_canonical():
-    content = httpRequestGetContent(digg_url)
+    content = get_http_content(digg_url)
     content_match = re.search(
         r'<link rel="canonical" href="(?P<url>[^"]+)', content)
     if content_match:
@@ -83,7 +83,7 @@ def check_item(item, root_item, org_url_start, _):
     statements = list()
     content = None
     if item['url'] not in checked_urls:
-        content = httpRequestGetContent(item['url'], True)
+        content = get_http_content(item['url'], True)
         time.sleep(1)
         checked_urls[item['url']] = content
     else:

tests/css_validator_w3c.py

@@ -16,8 +16,8 @@
 _local = gettext.gettext
 
 # DEFAULTS
-request_timeout = config.http_request_timeout
-useragent = config.useragent
+REQUEST_TIMEOUT = config.http_request_timeout
+USERAGENT = config.useragent
 css_review_group_errors = config.css_review_group_errors
 review_show_improvements_only = config.review_show_improvements_only
 sitespeed_use_docker = config.sitespeed_use_docker
@@ -27,12 +27,12 @@
     # If sitespeed timeout is not set in config.py this will be the default
     sitespeed_timeout = 600
 try:
-    use_cache = config.cache_when_possible
-    cache_time_delta = config.cache_time_delta
+    USE_CACHE = config.cache_when_possible
+    CACHE_TIME_DELTA = config.cache_time_delta
 except:
     # If cache_when_possible variable is not set in config.py this will be the default
-    use_cache = False
-    cache_time_delta = timedelta(hours=1)
+    USE_CACHE = False
+    CACHE_TIME_DELTA = timedelta(hours=1)
 
 global css_features
 global css_properties_doesnt_exist
@@ -306,7 +306,7 @@ def get_mdn_web_docs_css_features():
     css_features = {}
     css_functions = {}
 
-    html = httpRequestGetContent(
+    html = get_http_content(
         'https://developer.mozilla.org/en-US/docs/Web/CSS/Reference')
 
     soup = BeautifulSoup(html, 'lxml')

tests/email_validator.py

@@ -15,7 +15,7 @@
 import dns
 import config
 from models import Rating
-from tests.utils import dns_lookup, get_best_country_code, httpRequestGetContent, is_country_code_in_eu_or_on_exception_list
+from tests.utils import dns_lookup, get_best_country_code, get_http_content, is_country_code_in_eu_or_on_exception_list
 import gettext
 _local = gettext.gettext
 
@@ -152,13 +152,13 @@ def run_test(_, langCode, url):
         hostname, result_dict, _, _local)
     if rating.get_overall() == -1.0:
         # NO MX record found for domain, look for e-mail on website for alternative e-mail domain.
-        content = httpRequestGetContent(url, True)
+        content = get_http_content(url, True)
         time.sleep(1)
         result = search_for_email_domain(content)
         if result == None:
             interesting_urls = get_interesting_urls(content, url, 0)
             for interesting_url in interesting_urls:
-                content = httpRequestGetContent(interesting_url, True)
+                content = get_http_content(interesting_url, True)
                 result = search_for_email_domain(content)
                 if result != None:
                     break
@@ -362,7 +362,7 @@ def Validate_MTA_STS_Policy(_, rating, _local, hostname):
     rating += has_mta_sts_records_rating
 
     # https://mta-sts.example.com/.well-known/mta-sts.txt
-    content = httpRequestGetContent(
+    content = get_http_content(
         "https://mta-sts.{0}/.well-known/mta-sts.txt".format(hostname))
 
     has_mta_sts_txt_rating = Rating(_, review_show_improvements_only)

tests/energy_efficiency_websitecarbon.py

@@ -35,7 +35,7 @@ def run_test(_, langCode, url):
     print(_('TEXT_TEST_START').format(
         datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
 
-    result_json = httpRequestGetContent(
+    result_json = get_http_content(
         'https://api.websitecarbon.com/site?url={0}'.format(url))
     result_dict = json.loads(result_json)
 

tests/frontend_quality_yellow_lab_tools.py

@@ -9,7 +9,7 @@
 _local = gettext.gettext
 
 # DEFAULTS
-request_timeout = config.http_request_timeout
+REQUEST_TIMEOUT = config.http_request_timeout
 review_show_improvements_only = config.review_show_improvements_only
 time_sleep = config.webbkoll_sleep
 if time_sleep < 5:
@@ -58,23 +58,23 @@ def run_test(_, langCode, url, device='phone'):
 
         running_status = 'running'
         while running_status == 'running':
-            running_json = httpRequestGetContent(
+            running_json = get_http_content(
                 '{0}/api/runs/{1}'.format(ylt_server_address, test_id))
             running_info = json.loads(running_json)
             running_status = running_info['status']['statusCode']
             time.sleep(time_sleep)
 
         result_url = '{0}/api/results/{1}?exclude=toolsResults'.format(
             ylt_server_address, test_id)
-        result_json = httpRequestGetContent(result_url)
+        result_json = get_http_content(result_url)
     else:
         import subprocess
 
         # bashCommand = "yellowlabtools {0}".format(url)
         bashCommand = "node node_modules{1}yellowlabtools{1}bin{1}cli.js {0}".format(
             url, os.path.sep)
         process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-        output, error = process.communicate(timeout=request_timeout * 10)
+        output, error = process.communicate(timeout=REQUEST_TIMEOUT * 10)
 
         result_json = output
 

tests/html_validator_w3c.py

@@ -10,8 +10,8 @@
 _local = gettext.gettext
 
 # DEFAULTS
-request_timeout = config.http_request_timeout
-useragent = config.useragent
+REQUEST_TIMEOUT = config.http_request_timeout
+USERAGENT = config.useragent
 review_show_improvements_only = config.review_show_improvements_only
 sitespeed_use_docker = config.sitespeed_use_docker
 try:
@@ -20,12 +20,12 @@
     # If sitespeed timeout is not set in config.py this will be the default
     sitespeed_timeout = 600
 try:
-    use_cache = config.cache_when_possible
-    cache_time_delta = config.cache_time_delta
+    USE_CACHE = config.cache_when_possible
+    CACHE_TIME_DELTA = config.cache_time_delta
 except:
     # If cache_when_possible variable is not set in config.py this will be the default
-    use_cache = False
-    cache_time_delta = timedelta(hours=1)
+    USE_CACHE = False
+    CACHE_TIME_DELTA = timedelta(hours=1)
 
 
 def run_test(_, langCode, url):

tests/http_validator.py

@@ -32,9 +32,9 @@
 
 
 # DEFAULTS
-request_timeout = config.http_request_timeout
+REQUEST_TIMEOUT = config.http_request_timeout
 sitespeed_timeout = config.sitespeed_timeout
-useragent = config.useragent
+USERAGENT = config.useragent
 review_show_improvements_only = config.review_show_improvements_only
 sitespeed_use_docker = config.sitespeed_use_docker
 
@@ -44,12 +44,12 @@
     # If browser is not set in config.py this will be the default
     software_browser = 'chrome'
 try:
-    use_cache = config.cache_when_possible
-    cache_time_delta = config.cache_time_delta
+    USE_CACHE = config.cache_when_possible
+    CACHE_TIME_DELTA = config.cache_time_delta
 except:
     # If cache_when_possible variable is not set in config.py this will be the default
-    use_cache = False
-    cache_time_delta = timedelta(hours=1)
+    USE_CACHE = False
+    CACHE_TIME_DELTA = timedelta(hours=1)
 try:
     use_detailed_report = config.use_detailed_report
 except:
@@ -2363,9 +2363,9 @@ def has_weak_cipher(url, protocol_version):
     try:
         allow_redirects = False
 
-        headers = {'user-agent': useragent}
+        headers = {'user-agent': USERAGENT}
         a = session.get(url, verify=False, allow_redirects=allow_redirects,
-                        headers=headers, timeout=request_timeout)
+                        headers=headers, timeout=REQUEST_TIMEOUT)
 
         if a.status_code == 200 or a.status_code == 301 or a.status_code == 302 or a.status_code == 404:
             # print('is ok')
@@ -2451,9 +2451,9 @@ def has_insecure_cipher(url, protocol_version):
     try:
         allow_redirects = False
 
-        headers = {'user-agent': useragent}
+        headers = {'user-agent': USERAGENT}
         a = session.get(url, verify=False, allow_redirects=allow_redirects,
-                        headers=headers, timeout=request_timeout)
+                        headers=headers, timeout=REQUEST_TIMEOUT)
 
         if a.status_code == 200 or a.status_code == 301 or a.status_code == 302 or a.status_code == 404:
             # print('is ok')
@@ -2529,9 +2529,9 @@ def has_tls_version(url, validate_hostname, protocol_version):
     try:
         allow_redirects = False
 
-        headers = {'user-agent': useragent}
+        headers = {'user-agent': USERAGENT}
         session.get(url, verify=validate_hostname, allow_redirects=allow_redirects,
-                        headers=headers, timeout=request_timeout)
+                        headers=headers, timeout=REQUEST_TIMEOUT)
 
         return (True, 'is ok')
 

tests/lighthouse_base.py

@@ -4,14 +4,14 @@
 import json
 from tests.utils import *
 
-request_timeout = config.http_request_timeout
+REQUEST_TIMEOUT = config.http_request_timeout
 try:
-    use_cache = config.cache_when_possible
-    cache_time_delta = config.cache_time_delta
+    USE_CACHE = config.cache_when_possible
+    CACHE_TIME_DELTA = config.cache_time_delta
 except:
     # If cache_when_possible variable is not set in config.py this will be the default
-    use_cache = False
-    cache_time_delta = timedelta(hours=1)
+    USE_CACHE = False
+    CACHE_TIME_DELTA = timedelta(hours=1)
 
 def run_test(_, langCode, url, googlePageSpeedApiKey, strategy, category, review_show_improvements_only, lighthouse_use_api):
     """
@@ -165,15 +165,15 @@ def get_json_result(langCode, url, googlePageSpeedApiKey, strategy, category, li
         # print('pagespeed_api_request: {0}'.format(pagespeed_api_request))
 
         try:
-            get_content = httpRequestGetContent(pagespeed_api_request)
+            get_content = get_http_content(pagespeed_api_request)
             json_content = str_to_json(get_content, check_url)
             return json_content
         except:  # breaking and hoping for more luck with the next URL
             print(
                 'Error! Unfortunately the request for URL "{0}" failed, message:\n{1}'.format(
                     check_url, sys.exc_info()[0]))
             return
-    elif use_cache:
+    elif USE_CACHE:
         dir = Path(os.path.dirname(
             os.path.realpath(__file__)) + os.path.sep).parent
         try:
@@ -190,14 +190,14 @@ def get_json_result(langCode, url, googlePageSpeedApiKey, strategy, category, li
             bashCommand = "node node_modules{2}lighthouse{2}cli{2}index.js --output json --output-path {3} --locale {1} --form-factor {0} --chrome-flags=\"--headless\" --quiet".format(
                 strategy, langCode, os.path.sep, result_file)
             artifacts_file = os.path.join(cache_path, 'artifacts.json')
-            if os.path.exists(result_file) and not is_file_older_than(result_file, cache_time_delta):
+            if os.path.exists(result_file) and not is_file_older_than(result_file, CACHE_TIME_DELTA):
                 file_created_timestamp = os.path.getctime(result_file)
                 file_created_date = time.ctime(file_created_timestamp)
                 print('Cached entry found from {0}, using it instead of calling website again.'.format(
                     file_created_date))
                 with open(result_file, 'r', encoding='utf-8', newline='') as file:
                     return str_to_json('\n'.join(file.readlines()), check_url)
-            elif os.path.exists(artifacts_file) and not is_file_older_than(artifacts_file, cache_time_delta):
+            elif os.path.exists(artifacts_file) and not is_file_older_than(artifacts_file, CACHE_TIME_DELTA):
                 file_created_timestamp = os.path.getctime(artifacts_file)
                 file_created_date = time.ctime(file_created_timestamp)
                 print('Cached entry found from {0}, using it instead of calling website again.'.format(
@@ -209,7 +209,7 @@ def get_json_result(langCode, url, googlePageSpeedApiKey, strategy, category, li
             import subprocess
 
             process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-            output, error = process.communicate(timeout=request_timeout * 10)
+            output, error = process.communicate(timeout=REQUEST_TIMEOUT * 10)
             with open(result_file, 'r', encoding='utf-8', newline='') as file:
                 return str_to_json('\n'.join(file.readlines()), check_url)
         except:
@@ -221,7 +221,7 @@ def get_json_result(langCode, url, googlePageSpeedApiKey, strategy, category, li
         import subprocess
 
         process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
-        output, error = process.communicate(timeout=request_timeout * 10)
+        output, error = process.communicate(timeout=REQUEST_TIMEOUT * 10)
 
         get_content = output