Merge pull request Webperf-se#339 from cockroacher/i333

PyLint for Improved Code Quality - utils.py Webperf-se#333

Author: 7h3Rabbit

engines/sitemap.py

@@ -5,8 +5,7 @@
 import io
 from bs4 import BeautifulSoup
 from engines.utils import use_item
-from tests.utils import get_content_type, httpRequestGetContent, cache_time_delta
-from utils import merge_dicts
+from tests.utils import get_content_type, httpRequestGetContent, merge_dicts, cache_time_delta
 
 def read_sites(input_sitemap_url, input_skip, input_take):
     ignore_none_html = True

tests/http_validator.py

@@ -15,9 +15,8 @@
 import urllib
 import config
 from models import Rating
-from tests.utils import dns_lookup
+from tests.utils import dns_lookup, merge_dicts
 from tests.utils import *
-from utils import merge_dicts
 from tests.sitespeed_base import get_result
 import dns.name
 import dns.query

tests/utils.py

@@ -544,3 +544,41 @@ def get_friendly_url_name(_, url, request_index):
     except:
         return request_friendly_name
     return request_friendly_name
+
+def merge_dicts(dict1, dict2, sort, make_distinct):
+    if dict1 is None:
+        return dict2
+    if dict2 is None:
+        return dict1
+
+    for domain, value in dict2.items():
+        if domain in dict1:
+            type_of_value = type(value)
+            if type_of_value is dict:
+                for subkey, subvalue in value.items():
+                    if subkey in dict1[domain]:
+                        if isinstance(subvalue, dict):
+                            merge_dicts(
+                                dict1[domain][subkey],
+                                dict2[domain][subkey],
+                                sort,
+                                make_distinct)
+                        elif isinstance(subvalue, list):
+                            dict1[domain][subkey].extend(subvalue)
+                            if make_distinct:
+                                dict1[domain][subkey] = list(set(dict1[domain][subkey]))
+                            if sort:
+                                dict1[domain][subkey] = sorted(dict1[domain][subkey])
+                    else:
+                        dict1[domain][subkey] = dict2[domain][subkey]
+            elif type_of_value == list:
+                dict1[domain].extend(value)
+                if make_distinct:
+                    dict1[domain] = list(set(dict1[domain]))
+                if sort:
+                    dict1[domain] = sorted(dict1[domain])
+            elif type_of_value == int:
+                dict1[domain] = dict1[domain] + value
+        else:
+            dict1[domain] = value
+    return dict1