Skip to content

Commit

Permalink
Swap tokens (conda-forge#11085)
Browse files Browse the repository at this point in the history 
* tryt swapping tokens

* again
  • Loading branch information
@beckermr
beckermr authored Mar 19, 2020
1 parent 7e69151 commit f5b4529
Show file tree
Hide file tree
Showing 2 changed files with 105 additions and 46 deletions.
149 changes: 103 additions & 46 deletions .travis_scripts/create_feedstocks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
"""
Convert all recipes into feedstocks.
This script is to be run in a TravisCI context, with all secret environment variables defined (BINSTAR_TOKEN, GH_TOKEN)
This script is to be run in a TravisCI context, with all secret environment
variables defined (BINSTAR_TOKEN, GH_TOKEN)
Such as:
export GH_TOKEN=$(cat ~/.conda-smithy/github.token)
Expand All @@ -28,6 +30,8 @@


recipe_directory_name = 'recipes'


def list_recipes():
if os.path.isdir(recipe_directory_name):
recipes = os.listdir(recipe_directory_name)
Expand Down Expand Up @@ -84,9 +88,11 @@ def print_rate_limiting_info(gh, user):
print("GitHub API Rate Limit Info:")
print("---------------------------")
print("token: ", user)
print("Currently remaining {remaining} out of {total}.".format(remaining=gh_api_remaining, total=gh_api_total))
print("Currently remaining {remaining} out of {total}.".format(
remaining=gh_api_remaining, total=gh_api_total))
print("Will reset in {time}.".format(time=gh_api_reset_time))
print("")
return gh_api_remaining


def sleep_until_reset(gh):
Expand Down Expand Up @@ -116,7 +122,10 @@ def sleep_until_reset(gh):
if __name__ == '__main__':
exit_code = 0

is_merged_pr = (os.environ.get('TRAVIS_BRANCH') == 'master' and os.environ.get('TRAVIS_PULL_REQUEST') == 'false')
is_merged_pr = (
os.environ.get('TRAVIS_BRANCH') == 'master' and
os.environ.get('TRAVIS_PULL_REQUEST') == 'false'
)

smithy_conf = os.path.expanduser('~/.conda-smithy')
if not os.path.exists(smithy_conf):
Expand All @@ -134,20 +143,25 @@ def write_token(name, token):
if 'DRONE_TOKEN' in os.environ:
write_token('drone', os.environ['DRONE_TOKEN'])

gh_drone = Github(os.environ['GH_DRONE_TOKEN'])
gh_drone_remaining = print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')

gh = None
if 'GH_TOKEN' in os.environ:
write_token('github', os.environ['GH_TOKEN'])
gh = Github(os.environ['GH_TOKEN'])

# Get our initial rate limit info.
print_rate_limiting_info(gh, 'GH_TOKEN')
gh_remaining = print_rate_limiting_info(gh, 'GH_TOKEN')

# if we are out, exit early
# if sleep_until_reset(gh):
# sys.exit(1)

gh_drone = Github(os.environ['GH_DRONE_TOKEN'])
print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
# try the other token maybe?
# if gh_remaining < gh_drone_remaining and gh_remaining < 100:
# write_token('github', os.environ['GH_DRONE_TOKEN'])
# gh = Github(os.environ['GH_DRONE_TOKEN'])

owner_info = ['--organization', 'conda-forge']

Expand All @@ -158,34 +172,53 @@ def write_token(name, token):
feedstock_dir = os.path.join(feedstocks_dir, name + '-feedstock')
print('Making feedstock for {}'.format(name))
try:
subprocess.check_call(['conda', 'smithy', 'init', recipe_dir,
'--feedstock-directory', feedstock_dir])
subprocess.check_call(
['conda', 'smithy', 'init', recipe_dir,
'--feedstock-directory', feedstock_dir])
except subprocess.CalledProcessError:
traceback.print_exception(*sys.exc_info())
continue

if not is_merged_pr:
# We just want to check that conda-smithy is doing its thing without having any metadata issues.
# We just want to check that conda-smithy is doing its
# thing without having any metadata issues.
continue

feedstock_dirs.append([feedstock_dir, name, recipe_dir])

subprocess.check_call(['git', 'remote', 'add', 'upstream_with_token',
'https://conda-forge-manager:{}@github.com/conda-forge/{}-feedstock'.format(os.environ['GH_TOKEN'],
name)],
cwd=feedstock_dir)
subprocess.check_call([
'git', 'remote', 'add', 'upstream_with_token',
'https://conda-forge-manager:{}@github.com/'
'conda-forge/{}-feedstock'.format(
os.environ['GH_TOKEN'],
name
)
],
cwd=feedstock_dir
)
print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
# Sometimes we already have the feedstock created. We need to deal with that case.
# Sometimes we already have the feedstock created. We need to
# deal with that case.
if repo_exists(gh, 'conda-forge', name + '-feedstock'):
subprocess.check_call(['git', 'fetch', 'upstream_with_token'], cwd=feedstock_dir)
subprocess.check_call(['git', 'branch', '-m', 'master', 'old'], cwd=feedstock_dir)
subprocess.check_call(
['git', 'fetch', 'upstream_with_token'], cwd=feedstock_dir)
subprocess.check_call(
['git', 'branch', '-m', 'master', 'old'], cwd=feedstock_dir)
try:
subprocess.check_call(['git', 'checkout', '-b', 'master', 'upstream_with_token/master'], cwd=feedstock_dir)
subprocess.check_call(
[
'git', 'checkout', '-b', 'master',
'upstream_with_token/master'
],
cwd=feedstock_dir)
except subprocess.CalledProcessError:
# Sometimes, we have a repo, but there are no commits on it! Just catch that case.
subprocess.check_call(['git', 'checkout', '-b' 'master'], cwd=feedstock_dir)
# Sometimes, we have a repo, but there are no commits on
# it! Just catch that case.
subprocess.check_call(
['git', 'checkout', '-b' 'master'], cwd=feedstock_dir)
print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')
subprocess.check_call(['conda', 'smithy', 'register-github', feedstock_dir] + owner_info)
subprocess.check_call(
['conda', 'smithy', 'register-github', feedstock_dir] + owner_info)
print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')

from conda_smithy.ci_register import drone_sync
Expand All @@ -195,7 +228,8 @@ def write_token(name, token):
time.sleep(100) # actually wait
print_rate_limiting_info(gh_drone, 'GH_DRONE_TOKEN')

# Break the previous loop to allow the TravisCI registering to take place only once per function call.
# Break the previous loop to allow the TravisCI registering
# to take place only once per function call.
# Without this, intermittent failures to synch the TravisCI repos ensue.
# Hang on to any CI registration errors that occur and raise them at the end.
for num, (feedstock_dir, name, recipe_dir) in enumerate(feedstock_dirs):
Expand All @@ -209,9 +243,12 @@ def write_token(name, token):
# we fail the build so that people are aware that things did not clear.
try:
# Stop registering on appveyor temporarily
subprocess.check_call(['conda', 'smithy', 'register-ci', '--without-appveyor',
'--without-webservice', '--feedstock_directory', feedstock_dir] + owner_info)
subprocess.check_call(['conda', 'smithy', 'rerender'], cwd=feedstock_dir)
subprocess.check_call(
['conda', 'smithy', 'register-ci', '--without-appveyor',
'--without-webservice', '--feedstock_directory',
feedstock_dir] + owner_info)
subprocess.check_call(
['conda', 'smithy', 'rerender'], cwd=feedstock_dir)
except subprocess.CalledProcessError:
exit_code = 1
traceback.print_exception(*sys.exc_info())
Expand All @@ -221,26 +258,37 @@ def write_token(name, token):
for i in range(1, 13):
time.sleep(10)
print("Waiting for registration: {i} s".format(i=i*10))
subprocess.check_call(['git', 'commit', '--allow-empty', '-am', "Re-render the feedstock after CI registration."], cwd=feedstock_dir)
subprocess.check_call(
['git', 'commit', '--allow-empty', '-am',
"Re-render the feedstock after CI registration."], cwd=feedstock_dir)
for i in range(5):
try:
# Capture the output, as it may contain the GH_TOKEN.
out = subprocess.check_output(['git', 'push', 'upstream_with_token', 'HEAD:master'], cwd=feedstock_dir,
stderr=subprocess.STDOUT)
out = subprocess.check_output(
['git', 'push', 'upstream_with_token', 'HEAD:master'],
cwd=feedstock_dir,
stderr=subprocess.STDOUT)
break
except subprocess.CalledProcessError:
pass

# Likely another job has already pushed to this repo.
# Place our changes on top of theirs and try again.
out = subprocess.check_output(['git', 'fetch', 'upstream_with_token', 'master'], cwd=feedstock_dir,
stderr=subprocess.STDOUT)
out = subprocess.check_output(
['git', 'fetch', 'upstream_with_token', 'master'],
cwd=feedstock_dir,
stderr=subprocess.STDOUT)
try:
subprocess.check_call(['git', 'rebase', 'upstream_with_token/master', 'master'], cwd=feedstock_dir)
subprocess.check_call(
['git', 'rebase', 'upstream_with_token/master', 'master'],
cwd=feedstock_dir)
except subprocess.CalledProcessError:
# Handle rebase failure by choosing the changes in `master`.
subprocess.check_call(['git', 'checkout', 'master', '--', '.'], cwd=feedstock_dir)
subprocess.check_call(['git', 'rebase', '--continue'], cwd=feedstock_dir)
subprocess.check_call(
['git', 'checkout', 'master', '--', '.'],
cwd=feedstock_dir)
subprocess.check_call(
['git', 'rebase', '--continue'], cwd=feedstock_dir)

# Remove this recipe from the repo.
if is_merged_pr:
Expand All @@ -258,27 +306,33 @@ def write_token(name, token):
# Finish quietly.
pass

# Parse `git status --porcelain` to handle some merge conflicts and generate the removed recipe list.
changed_files = subprocess.check_output(['git', 'status', '--porcelain', recipe_directory_name],
universal_newlines=True)
# Parse `git status --porcelain` to handle some merge conflicts and
# generate the removed recipe list.
changed_files = subprocess.check_output(
['git', 'status', '--porcelain', recipe_directory_name],
universal_newlines=True)
changed_files = changed_files.splitlines()

# Add all files from AU conflicts. They are new files that we weren't tracking previously.
# Add all files from AU conflicts. They are new files that we
# weren't tracking previously.
# Adding them resolves the conflict and doesn't actually add anything to the index.
new_file_conflicts = filter(lambda _: _.startswith("AU "), changed_files)
new_file_conflicts = map(lambda _ : _.replace("AU", "", 1).lstrip(), new_file_conflicts)
new_file_conflicts = map(
lambda _: _.replace("AU", "", 1).lstrip(), new_file_conflicts)
for each_new_file in new_file_conflicts:
subprocess.check_call(['git', 'add', each_new_file])

# Generate a fresh listing of recipes removed.
#
# * Each line we get back is a change to a file in the recipe directory.
# * We narrow the list down to recipes that are staged for deletion (ignores examples).
# * We narrow the list down to recipes that are staged for deletion
# (ignores examples).
# * Then we clean up the list so that it only has the recipe names.
removed_recipes = filter(lambda _: _.startswith("D "), changed_files)
removed_recipes = map(lambda _ : _.replace("D", "", 1).lstrip(), removed_recipes)
removed_recipes = map(lambda _ : os.path.relpath(_, recipe_directory_name), removed_recipes)
removed_recipes = map(lambda _ : _.split(os.path.sep)[0], removed_recipes)
removed_recipes = map(lambda _: _.replace("D", "", 1).lstrip(), removed_recipes)
removed_recipes = map(
lambda _: os.path.relpath(_, recipe_directory_name), removed_recipes)
removed_recipes = map(lambda _: _.split(os.path.sep)[0], removed_recipes)
removed_recipes = sorted(set(removed_recipes))

# Commit any removed packages.
Expand All @@ -290,14 +344,17 @@ def write_token(name, token):
msg += ' [ci skip]'
if is_merged_pr:
# Capture the output, as it may contain the GH_TOKEN.
out = subprocess.check_output(['git', 'remote', 'add', 'upstream_with_token',
'https://conda-forge-manager:{}@github.com/conda-forge/staged-recipes'.format(os.environ['GH_TOKEN'])],
stderr=subprocess.STDOUT)
out = subprocess.check_output(
['git', 'remote', 'add', 'upstream_with_token',
'https://conda-forge-manager:{}@github.com/'
'conda-forge/staged-recipes'.format(os.environ['GH_TOKEN'])],
stderr=subprocess.STDOUT)
subprocess.check_call(['git', 'commit', '-m', msg])
# Capture the output, as it may contain the GH_TOKEN.
branch = os.environ.get('TRAVIS_BRANCH')
out = subprocess.check_output(['git', 'push', 'upstream_with_token', 'HEAD:%s' % branch],
stderr=subprocess.STDOUT)
out = subprocess.check_output(
['git', 'push', 'upstream_with_token', 'HEAD:%s' % branch],
stderr=subprocess.STDOUT)
else:
print('Would git commit, with the following message: \n {}'.format(msg))

Expand Down
2 changes: 2 additions & 0 deletions setup.cfg
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[flake8]
max-line-length=88

0 comments on commit f5b4529

Please sign in to comment.