diff --git a/doc/make.bat b/doc/make.bat old mode 100644 new mode 100755 diff --git a/src/rosdep2/gbpdistro_support.py b/src/rosdep2/gbpdistro_support.py index fd90ae096..5643e5249 100644 --- a/src/rosdep2/gbpdistro_support.py +++ b/src/rosdep2/gbpdistro_support.py @@ -1,7 +1,4 @@ -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen +from rosdep2.shell_utils import FakeURLOpener as urlopen import yaml try: import urlparse diff --git a/src/rosdep2/platforms/source.py b/src/rosdep2/platforms/source.py index 3ba17fedc..579b7cb3d 100644 --- a/src/rosdep2/platforms/source.py +++ b/src/rosdep2/platforms/source.py @@ -30,12 +30,11 @@ from __future__ import print_function import os +from rosdep2.shell_utils import FakeURLOpener as urlopen try: - from urllib.request import urlopen from urllib.request import urlretrieve from urllib.error import URLError except ImportError: - from urllib2 import urlopen from urllib import urlretrieve from urllib2 import URLError import hashlib diff --git a/src/rosdep2/rep3.py b/src/rosdep2/rep3.py index 71b2f56f4..1f5d3705c 100644 --- a/src/rosdep2/rep3.py +++ b/src/rosdep2/rep3.py @@ -25,10 +25,7 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen +from rosdep2.shell_utils import FakeURLOpener as urlopen import yaml import warnings diff --git a/src/rosdep2/rosdistrohelper.py b/src/rosdep2/rosdistrohelper.py index 9b46e9669..552f01243 100644 --- a/src/rosdep2/rosdistrohelper.py +++ b/src/rosdep2/rosdistrohelper.py @@ -28,6 +28,9 @@ # Author Paul Mathieu/paul@osrfoundation.org import rosdistro +from rosdep2.shell_utils import FakeURLOpener + +rosdistro.load_url = lambda url, **kwargs: FakeURLOpener(url).read(**kwargs) class PreRep137Warning(UserWarning): diff --git a/src/rosdep2/shell_utils.py b/src/rosdep2/shell_utils.py index 8dde39155..85b58c7b8 100644 --- a/src/rosdep2/shell_utils.py +++ b/src/rosdep2/shell_utils.py @@ -103,3 +103,50 @@ def create_tempfile_from_string_and_execute(string_script, path=None, exec_fn=No rd_debug('Return code was: %s' % (result)) return result == 0 + + +class FakeURLOpener(object): + url_map = { + 'github.com': 'hub.fastgit.org', + 'raw.githubusercontent.com': 'raw.fastgit.org', + } + def __init__(self, url, **kwargs): + self.url = url if isinstance(url, str) else url.get_full_url() + self.lftp_bin = read_stdout(['which', 'lftp']).strip() or None + assert self.lftp_bin is not None, 'lftp not found in PATH, please install it.' + + def read(self, skip_decode=True): + with tempfile.NamedTemporaryFile() as tmp_file: + _, error = self.lftp(tmp_file.name) + if error: + raise Exception('lftp failed: %s' % error) + contents = tmp_file.read() + if isinstance(contents, str) or skip_decode: + return contents + else: + return contents.decode('utf-8') + + def close(self): + pass + + def lftp(self, dst_path): + """Download file over bad connection network env with lftp""" + url = self.url + for key, value in self.url_map.items(): + if key in self.url: + url = self.url.replace(key, value) + break + + arg_list = [ + self.lftp_bin, + '-e', + """ + set net:idle 10; + set net:max-retries 0; + set net:reconnect-interval-base 3; + set net:reconnect-interval-max 3; + set http:user-agent 'rosdep/1.0'; + pget -n 10 -c {} -o {}; exit""".format(url, dst_path), + ] + # print(url) + return read_stdout(arg_list, capture_stderr=True) diff --git a/src/rosdep2/sources_list.py b/src/rosdep2/sources_list.py index 72bdc7244..f7187bb4f 100644 --- a/src/rosdep2/sources_list.py +++ b/src/rosdep2/sources_list.py @@ -32,12 +32,11 @@ import os import sys import yaml +from rosdep2.shell_utils import FakeURLOpener as urlopen try: - from urllib.request import urlopen from urllib.error import URLError import urllib.request as request except ImportError: - from urllib2 import urlopen from urllib2 import URLError import urllib2 as request try: @@ -512,6 +511,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, print('Skip end-of-life distro "%s"' % dist_name) continue print('Add distro "%s"' % dist_name) + # import pdb; pdb.set_trace() rds = RosDistroSource(dist_name) rosdep_data = get_gbprepo_as_rosdep_data(dist_name) # Store Python version from REP153