From a79ac09758aea78de70a694189e8730cf615c351 Mon Sep 17 00:00:00 2001 From: James Goppert Date: Thu, 15 Jun 2017 22:00:01 -0400 Subject: [PATCH] Enable github private repos. --- .gitignore | 1 + bloom/commands/release.py | 33 +++++++++++++++--------------- bloom/github.py | 42 ++++++++++++++++++++------------------- bloom/util.py | 28 +++++++++++++++++++++++--- 4 files changed, 64 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 6c106ead..04eb24ae 100755 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ MANIFEST *.DS_Store deb_dist .fuse_* +*.swp diff --git a/bloom/commands/release.py b/bloom/commands/release.py index 9147da31..511b57ce 100644 --- a/bloom/commands/release.py +++ b/bloom/commands/release.py @@ -36,7 +36,6 @@ import argparse import atexit -import base64 import datetime import difflib import getpass @@ -73,8 +72,6 @@ from bloom.git import inbranch from bloom.git import ls_tree -from bloom.github import auth_header_from_basic_auth -from bloom.github import auth_header_from_oauth_token from bloom.github import Github from bloom.github import GithubException @@ -104,6 +101,7 @@ from bloom.util import safe_input from bloom.util import temporary_directory from bloom.util import to_unicode +from bloom.util import auth_header try: import vcstools @@ -395,7 +393,7 @@ def get_relative_distribution_file_path(distro): def generate_release_tag(distro): - return ('release/%s/{package}/{version}' % distro).encode('utf-8') + return 'release/%s/{package}/{version}' % distro def generate_ros_distro_diff(track, repository, distro, override_release_repository_url): @@ -409,26 +407,26 @@ def generate_ros_distro_diff(track, repository, distro, override_release_reposit track_dict = get_tracks_dict_raw()['tracks'][track] last_version = track_dict['last_version'] release_inc = track_dict['release_inc'] - version = '{0}-{1}'.format(last_version, release_inc).encode('utf-8') + version = '{0}-{1}'.format(last_version, release_inc) # Create a repository if there isn't already one if repository not in distribution_dict['repositories']: distribution_dict['repositories'][repository] = {} # Create a release entry if there isn't already one if 'release' not in distribution_dict['repositories'][repository]: - distribution_dict['repositories'][repository]['release'.encode('utf-8')] = { - 'url'.encode('utf-8'): override_release_repository_url or _user_provided_release_url + distribution_dict['repositories'][repository]['release'] = { + 'url': override_release_repository_url or _user_provided_release_url } # Update the repository repo = distribution_dict['repositories'][repository]['release'] # Consider the override if override_release_repository_url is not None: - repo['url'.encode('utf-8')] = override_release_repository_url + repo['url'] = override_release_repository_url if 'tags' not in repo: - repo['tags'.encode('utf-8')] = {} - repo['tags']['release'.encode('utf-8')] = generate_release_tag(distro) - repo['version'.encode('utf-8')] = version + repo['tags'] = {} + repo['tags']['release'] = generate_release_tag(distro) + repo['version'] = version if 'packages' not in repo: - repo['packages'.encode('utf-8')] = [] + repo['packages'] = [] for path, pkg in packages.items(): if pkg.name not in repo['packages']: repo['packages'].append(pkg.name) @@ -577,8 +575,10 @@ def get_repository_info_from_user(url_type, defaults=None): distro_file_raw = load_url_to_file_handle(get_distribution_file_url(distro)).read() if distro_file_raw != distro_dump: # Calculate the diff - udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(), - fromfile=distro_file_name, tofile=distro_file_name) + udiff = difflib.unified_diff( + list(map(str, distro_file_raw.splitlines())), + list(map(str, distro_dump.splitlines())), + fromfile=distro_file_name, tofile=distro_file_name) temp_dir = tempfile.mkdtemp() udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch') udiff_raw = '' @@ -625,7 +625,6 @@ def get_repository_info_from_user(url_type, defaults=None): def get_gh_info(url): - from urlparse import urlparse o = urlparse(url) if 'raw.github.com' not in o.netloc and 'raw.githubusercontent.com' not in o.netloc: return None, None, None, None @@ -651,7 +650,7 @@ def get_github_interface(quiet=False): token = config.get('oauth_token', None) username = config.get('github_user', None) if token and username: - return Github(username, auth=auth_header_from_oauth_token(token), token=token) + return Github(username, auth=auth_header(token=token), token=token) if not os.path.isdir(os.path.dirname(oauth_config_path)): os.makedirs(os.path.dirname(oauth_config_path)) if quiet: @@ -679,7 +678,7 @@ def get_github_interface(quiet=False): if not password: error("No password was given, aborting.") return None - gh = Github(username, auth=auth_header_from_basic_auth(username, password)) + gh = Github(username, auth=auth_header(username=username, password=password)) try: token = gh.create_new_bloom_authorization(update_auth=True) with open(oauth_config_path, 'a') as f: diff --git a/bloom/github.py b/bloom/github.py index b987c84c..b5f35621 100644 --- a/bloom/github.py +++ b/bloom/github.py @@ -41,30 +41,24 @@ import datetime import json import socket +import os -from urlparse import urlunsplit -from urllib import urlencode try: # Python2 + from urlparse import urlencode, urlunsplit from urllib2 import HTTPError from urllib2 import URLError from urllib2 import Request, urlopen except ImportError: # Python3 + from urllib.parse import urlencode, urlunsplit from urllib.error import HTTPError from urllib.error import URLError from urllib.request import Request, urlopen import bloom - - -def auth_header_from_basic_auth(user, password): - return "Basic {0}".format(base64.b64encode('{0}:{1}'.format(user, password))) - - -def auth_header_from_oauth_token(token): - return "token " + token +from bloom.util import auth_header, GITHUB_USER, GITHUB_PASSWORD def get_bloom_headers(auth=None): @@ -86,8 +80,11 @@ def do_github_post_req(path, data=None, auth=None, site='api.github.com'): if data is None: request = Request(url, headers=headers) # GET else: - request = Request(url, data=json.dumps(data), headers=headers) # POST + request = Request(url, data=json.dumps(data).encode('utf-8'), headers=headers) # POST + if GITHUB_USER and GITHUB_PASSWORD: + request.add_header('Authorization', auth_header( + username=GITHUB_USER, password=GITHUB_PASSWORD)) try: response = urlopen(request, timeout=120) except HTTPError as e: @@ -125,13 +122,13 @@ def create_new_bloom_authorization(self, note=None, note_url=None, scopes=None, "note_url": 'http://bloom.readthedocs.org/' if note_url is None else note_url } resp = do_github_post_req('/authorizations', payload, self.auth) - resp_data = json.loads(resp.read()) + resp_data = json.loads(str(resp.read())) resp_code = '{0}'.format(resp.getcode()) if resp_code not in ['201', '202'] or 'token' not in resp_data: raise GithubException("Failed to create a new oauth authorization", resp) token = resp_data['token'] if update_auth: - self.auth = auth_header_from_oauth_token(token) + self.auth = auth_header(token=token) self.token = token return token @@ -140,7 +137,7 @@ def get_repo(self, owner, repo): if '{0}'.format(resp.getcode()) not in ['200']: raise GithubException( "Failed to get information for repository '{owner}/{repo}'".format(**locals()), resp) - resp_data = json.loads(resp.read()) + resp_data = json.loads(str(resp.read())) return resp_data def list_repos(self, user, start_page=None): @@ -152,7 +149,7 @@ def list_repos(self, user, start_page=None): if '{0}'.format(resp.getcode()) not in ['200']: raise GithubException( "Failed to list repositories for user '{user}' using url '{url}'".format(**locals()), resp) - new_repos = json.loads(resp.read()) + new_repos = json.loads(str(resp.read())) if not new_repos: return repos repos.extend(new_repos) @@ -165,7 +162,8 @@ def get_branch(self, owner, repo, branch): raise GithubException("Failed to get branch information for '{branch}' on '{owner}/{repo}' using '{url}'" .format(**locals()), resp) - return json.loads(resp.read()) + txt = resp.read().decode('utf-8') + return json.loads(txt) def list_branches(self, owner, repo, start_page=None): page = start_page or 1 @@ -176,7 +174,8 @@ def list_branches(self, owner, repo, start_page=None): if '{0}'.format(resp.getcode()) not in ['200']: raise GithubException( "Failed to list branches for '{owner}/{repo}' using url '{url}'".format(**locals()), resp) - new_branches = json.loads(resp.read()) + txt = resp.read().decode('utf-8') + new_branches = json.loads(txt) if not new_branches: return branches branches.extend(new_branches) @@ -187,7 +186,8 @@ def create_fork(self, parent_org, parent_repo): if '{0}'.format(resp.getcode()) not in ['200', '202']: raise GithubException( "Failed to create a fork of '{parent_org}/{parent_repo}'".format(**locals()), resp) - return json.loads(resp.read()) + txt = resp.read().decode('utf-8') + return json.loads(txt) def list_forks(self, org, repo, start_page=None): page = start_page or 1 @@ -198,7 +198,8 @@ def list_forks(self, org, repo, start_page=None): if '{0}'.format(resp.getcode()) not in ['200', '202']: raise GithubException( "Failed to list forks of '{org}/{repo}'".format(**locals()), resp) - new_forks = json.loads(resp.read()) + txt = resp.read().decode('utf-8') + new_forks = json.loads(txt) if not new_forks: return forks forks.extend(new_forks) @@ -214,5 +215,6 @@ def create_pull_request(self, org, repo, branch, fork_org, fork_branch, title, b resp = do_github_post_req('/repos/{org}/{repo}/pulls'.format(**locals()), data, self.auth) if '{0}'.format(resp.getcode()) not in ['200', '201']: raise GithubException("Failed to create pull request", resp) - resp_json = json.loads(resp.read()) + txt = resp.read().decode('utf-8') + resp_json = json.loads(txt) return resp_json['html_url'] diff --git a/bloom/util.py b/bloom/util.py index 07fdd534..5d077f7b 100755 --- a/bloom/util.py +++ b/bloom/util.py @@ -38,6 +38,7 @@ import argparse import os +import base64 import shutil import socket import sys @@ -48,12 +49,12 @@ # Python2 from urllib2 import HTTPError from urllib2 import URLError - from urllib2 import urlopen + from urllib2 import urlopen, Request except ImportError: # Python3 from urllib.error import HTTPError from urllib.error import URLError - from urllib.request import urlopen + from urllib.request import urlopen, Request from email.utils import formatdate @@ -78,6 +79,23 @@ from bloom.logging import sanitize from bloom.logging import warning + +GITHUB_USER = os.getenv('GITHUB_USER', None) +GITHUB_PASSWORD = os.getenv('GITHUB_PASSWORD', None) + + +def auth_header(username=None, password=None, token=None): + if username and password: + if sys.version_info > (3, 0): + userandpass = base64.b64encode(bytes('%s:%s' % (username, password), 'utf-8')) + else: + userandpass = base64.b64encode('%s:%s' % (username, password)) + userandpass = userandpass.decode('ascii') + return 'Basic %s' % userandpass + elif token: + return 'token %s' % token + + try: to_unicode = unicode except NameError: @@ -195,8 +213,12 @@ def load_url_to_file_handle(url, retry=2, retry_period=1, timeout=10): :param timeout: timeout for opening the URL in seconds :type timeout: float """ + req = Request(url) + if GITHUB_USER and GITHUB_PASSWORD: + req.add_header('Authorization', auth_header( + username=GITHUB_USER, password=GITHUB_PASSWORD)) try: - fh = urlopen(url, timeout=timeout) + fh = urlopen(req, timeout=timeout) except HTTPError as e: if e.code == 503 and retry: time.sleep(retry_period)