Skip to content

Commit 6974800

Browse files
committed
Auto merge of #33288 - cyplo:32834_retry_download, r=alexcrichton
Get a file during bootstrap to a temp location first. When downloading a file in the bootstrap phase - get it to a temp location first. Verify it there and only if downloaded properly move it to the `cache` directory. This should prevent `make` being stuck if the download was interrupted or otherwise corrupted, as per discussion in #32834 The temporary files are deleted in case of an exception. I was looking for some unit/integration tests around this and couldn't find any - presumably because this is being tested by just Travis launching it ? Let me know if it would be good to try to write tests around this. Thanks !
2 parents ebe6da3 + 6bce110 commit 6974800

File tree

1 file changed

+38
-11
lines changed

1 file changed

+38
-11
lines changed

src/bootstrap/bootstrap.py

+38-11
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,46 @@
1616
import subprocess
1717
import sys
1818
import tarfile
19+
import tempfile
20+
1921

2022
def get(url, path, verbose=False):
21-
print("downloading " + url)
2223
sha_url = url + ".sha256"
23-
sha_path = path + ".sha256"
24-
for _url, _path in ((url, path), (sha_url, sha_path)):
25-
# see http://serverfault.com/questions/301128/how-to-download
26-
if sys.platform == 'win32':
27-
run(["PowerShell.exe", "/nologo", "-Command",
28-
"(New-Object System.Net.WebClient)"
29-
".DownloadFile('{}', '{}')".format(_url, _path)],
30-
verbose=verbose)
31-
else:
32-
run(["curl", "-o", _path, _url], verbose=verbose)
24+
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
25+
temp_path = temp_file.name
26+
with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
27+
sha_path = sha_file.name
28+
29+
try:
30+
download(sha_path, sha_url, verbose)
31+
download(temp_path, url, verbose)
32+
verify(temp_path, sha_path, verbose)
33+
print("moving " + temp_path + " to " + path)
34+
shutil.move(temp_path, path)
35+
finally:
36+
delete_if_present(sha_path)
37+
delete_if_present(temp_path)
38+
39+
40+
def delete_if_present(path):
41+
if os.path.isfile(path):
42+
print("removing " + path)
43+
os.unlink(path)
44+
45+
46+
def download(path, url, verbose):
47+
print("downloading " + url + " to " + path)
48+
# see http://serverfault.com/questions/301128/how-to-download
49+
if sys.platform == 'win32':
50+
run(["PowerShell.exe", "/nologo", "-Command",
51+
"(New-Object System.Net.WebClient)"
52+
".DownloadFile('{}', '{}')".format(url, path)],
53+
verbose=verbose)
54+
else:
55+
run(["curl", "-o", path, url], verbose=verbose)
56+
57+
58+
def verify(path, sha_path, verbose):
3359
print("verifying " + path)
3460
with open(path, "rb") as f:
3561
found = hashlib.sha256(f.read()).hexdigest()
@@ -43,6 +69,7 @@ def get(url, path, verbose=False):
4369
raise RuntimeError(err)
4470
sys.exit(err)
4571

72+
4673
def unpack(tarball, dst, verbose=False, match=None):
4774
print("extracting " + tarball)
4875
fname = os.path.basename(tarball).replace(".tar.gz", "")

0 commit comments

Comments
 (0)