rosdep for CN users

src/rosdep2/gbpdistro_support.py

@@ -1,7 +1,4 @@
-try:
-    from urllib.request import urlopen
-except ImportError:
-    from urllib2 import urlopen
+from rosdep2.shell_utils import FakeURLOpener as urlopen
 import yaml
 try:
     import urlparse

src/rosdep2/platforms/source.py

@@ -30,12 +30,11 @@
 from __future__ import print_function
 
 import os
+from rosdep2.shell_utils import FakeURLOpener as urlopen
 try:
-    from urllib.request import urlopen
     from urllib.request import urlretrieve
     from urllib.error import URLError
 except ImportError:
-    from urllib2 import urlopen
     from urllib import urlretrieve
     from urllib2 import URLError
 import hashlib

src/rosdep2/rep3.py

@@ -25,10 +25,7 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-try:
-    from urllib.request import urlopen
-except ImportError:
-    from urllib2 import urlopen
+from rosdep2.shell_utils import FakeURLOpener as urlopen
 import yaml
 import warnings
 

src/rosdep2/rosdistrohelper.py

@@ -28,6 +28,9 @@
 # Author Paul Mathieu/[email protected]
 
 import rosdistro
+from rosdep2.shell_utils import FakeURLOpener
+
+rosdistro.load_url = lambda url, **kwargs: FakeURLOpener(url).read(**kwargs)
 
 
 class PreRep137Warning(UserWarning):

src/rosdep2/shell_utils.py

@@ -103,3 +103,50 @@ def create_tempfile_from_string_and_execute(string_script, path=None, exec_fn=No
 
     rd_debug('Return code was: %s' % (result))
     return result == 0
+
+
+class FakeURLOpener(object):
+    url_map = {
+        'github.com': 'hub.fastgit.org',
+        'raw.githubusercontent.com': 'raw.fastgit.org',
+    }
+    def __init__(self, url, **kwargs):
+        self.url = url if isinstance(url, str) else url.get_full_url()
+        self.lftp_bin = read_stdout(['which', 'lftp']).strip() or None
+        assert self.lftp_bin is not None, 'lftp not found in PATH, please install it.'
+
+    def read(self, skip_decode=True):
+        with tempfile.NamedTemporaryFile() as tmp_file:
+            _, error = self.lftp(tmp_file.name)
+            if error:
+                raise Exception('lftp failed: %s' % error)
+            contents = tmp_file.read()
+            if isinstance(contents, str) or skip_decode:
+                return contents
+            else:
+                return contents.decode('utf-8')
+
+    def close(self):
+        pass
+
+    def lftp(self, dst_path):
+        """Download file over bad connection network env with lftp"""
+        url = self.url
+        for key, value in self.url_map.items():
+            if key in self.url:
+                url = self.url.replace(key, value)
+                break
+
+        arg_list = [
+            self.lftp_bin,
+            '-e',
+            """
+            set net:idle 10;
+            set net:max-retries 0;
+            set net:reconnect-interval-base 3;
+            set net:reconnect-interval-max 3;
+            set http:user-agent 'rosdep/1.0';
+            pget -n 10 -c {} -o {}; exit""".format(url, dst_path),
+        ]
+        # print(url)
+        return read_stdout(arg_list, capture_stderr=True)

src/rosdep2/sources_list.py

@@ -32,12 +32,11 @@
 import os
 import sys
 import yaml
+from rosdep2.shell_utils import FakeURLOpener as urlopen
 try:
-    from urllib.request import urlopen
     from urllib.error import URLError
     import urllib.request as request
 except ImportError:
-    from urllib2 import urlopen
     from urllib2 import URLError
     import urllib2 as request
 try:
@@ -512,6 +511,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
                     print('Skip end-of-life distro "%s"' % dist_name)
                     continue
         print('Add distro "%s"' % dist_name)
+        # import pdb; pdb.set_trace()
         rds = RosDistroSource(dist_name)
         rosdep_data = get_gbprepo_as_rosdep_data(dist_name)
         # Store Python version from REP153