12
12
import re
13
13
import os
14
14
from urllib .parse import ParseResult , urlparse , urlunparse
15
+ import gettext
15
16
import requests
16
17
import IP2Location
17
18
import dns
20
21
import dns .dnssec
21
22
import dns .exception
22
23
import dns .name
23
- import gettext
24
24
25
25
def get_config_or_default (name ):
26
26
"""
@@ -79,10 +79,8 @@ def get_translation(module_name, lang_code):
79
79
Returns:
80
80
function: The gettext() function for the specified language.
81
81
"""
82
- language = gettext .translation (
83
- module_name ,
84
- localedir = 'locales' ,
85
- languages = [lang_code ])
82
+ language = gettext .translation (module_name ,
83
+ localedir = 'locales' , languages = [lang_code ])
86
84
return language .gettext
87
85
88
86
@@ -110,14 +108,9 @@ def change_url_to_test_url(url, test_name):
110
108
else :
111
109
new_query = f'webperf-core={ test_name } &' + o .query
112
110
o2 = ParseResult (
113
- scheme = o .scheme ,
114
- netloc = o .netloc ,
115
- path = o .path ,
116
- params = o .params ,
117
- query = new_query ,
118
- fragment = o .fragment )
119
- url2 = urlunparse (o2 )
120
- return url2
111
+ scheme = o .scheme , netloc = o .netloc , path = o .path ,
112
+ params = o .params , query = new_query , fragment = o .fragment )
113
+ return urlunparse (o2 )
121
114
122
115
123
116
def is_file_older_than (file , delta ):
@@ -173,8 +166,7 @@ def get_cache_path_for_rule(url, cache_key_rule):
173
166
if not os .path .exists (hostname_path ):
174
167
os .makedirs (hostname_path )
175
168
176
- cache_key = cache_key_rule .format (
177
- hashlib .sha512 (url .encode ()).hexdigest ())
169
+ cache_key = cache_key_rule .format (hashlib .sha512 (url .encode ()).hexdigest ())
178
170
cache_path = os .path .join (folder , hostname , cache_key )
179
171
180
172
return cache_path
@@ -191,9 +183,7 @@ def get_cache_path_for_folder(url):
191
183
Returns:
192
184
str: The generated cache path.
193
185
"""
194
-
195
186
cache_key_rule = '{0}'
196
-
197
187
return get_cache_path_for_rule (url , cache_key_rule )
198
188
199
189
@@ -212,15 +202,12 @@ def get_cache_path_for_file(url, use_text_instead_of_content):
212
202
Returns:
213
203
str: The generated cache path.
214
204
"""
215
-
216
205
file_ending = '.tmp'
217
206
if USE_CACHE :
218
207
file_ending = '.cache'
219
-
220
208
cache_key_rule = '{0}.txt.utf-8' + file_ending
221
209
if not use_text_instead_of_content :
222
210
cache_key_rule = '{0}.bytes' + file_ending
223
-
224
211
return get_cache_path_for_rule (url , cache_key_rule )
225
212
226
213
@@ -243,17 +230,15 @@ def get_cache_file(url, use_text_instead_of_content, time_delta):
243
230
If the cache file does not exist or is too old, None is returned.
244
231
245
232
Notes:
246
- - The function uses the get_cache_path_for_file function to determine the path of the cache file.
233
+ - The function uses the get_cache_path_for_file function
234
+ to determine the path of the cache file.
247
235
- If USE_CACHE is False, the function always returns None.
248
236
"""
249
237
cache_path = get_cache_path_for_file (url , use_text_instead_of_content )
250
-
251
238
if not os .path .exists (cache_path ):
252
239
return None
253
-
254
240
if USE_CACHE and is_file_older_than (cache_path , time_delta ):
255
241
return None
256
-
257
242
if use_text_instead_of_content :
258
243
with open (cache_path , 'r' , encoding = 'utf-8' , newline = '' ) as file :
259
244
return '\n ' .join (file .readlines ())
@@ -277,13 +262,10 @@ def has_cache_file(url, use_text_instead_of_content, time_delta):
277
262
False otherwise.
278
263
"""
279
264
cache_path = get_cache_path_for_file (url , use_text_instead_of_content )
280
-
281
265
if not os .path .exists (cache_path ):
282
266
return False
283
-
284
267
if USE_CACHE and is_file_older_than (cache_path , time_delta ):
285
268
return False
286
-
287
269
return True
288
270
289
271
@@ -316,18 +298,15 @@ def clean_cache_files():
316
298
if os .path .exists (base_directory ):
317
299
shutil .rmtree (base_directory )
318
300
return
319
-
320
301
file_ending = '.cache'
321
302
folder = 'cache'
322
-
323
303
base_directory = os .path .join (Path (os .path .dirname (
324
304
os .path .realpath (__file__ )) + os .path .sep ).parent , folder )
325
305
326
306
if not os .path .exists (base_directory ):
327
307
return
328
308
329
309
print (f'Cleaning { file_ending [1 :]} files...' )
330
-
331
310
subdirs = os .listdir (base_directory )
332
311
print (len (subdirs ), f'file and folders in { folder } folder.' )
333
312
cache_files = 0
@@ -406,9 +385,7 @@ def get_http_content(url, allow_redirects=False, use_text_instead_of_content=Tru
406
385
return content
407
386
408
387
headers = {'user-agent' : USERAGENT }
409
-
410
388
hostname = urlparse (url ).hostname
411
-
412
389
if hostname == 'api.github.com' and GITHUB_APIKEY is not None :
413
390
headers ['authorization' ] = f'Bearer { GITHUB_APIKEY } '
414
391
a = requests .get (url , allow_redirects = allow_redirects ,
@@ -842,8 +819,7 @@ def get_country_code_from_ip2location(ip_address):
842
819
except Exception : # pylint: disable=broad-exception-caught
843
820
return ''
844
821
if hasattr (rec , 'country_short' ):
845
- countrycode = rec .country_short
846
- return countrycode
822
+ return rec .country_short
847
823
return ''
848
824
849
825
0 commit comments