Skip to content

Commit

Permalink
Merge remote-tracking branch 'skeleton/main' into prepare-release
Browse files Browse the repository at this point in the history
  • Loading branch information
@pombredanne
pombredanne committed May 9, 2022
2 parents 99d274d + 6a3c5b0 commit 69bfc1b
Show file tree
Hide file tree
Showing 6 changed files with 488 additions and 649 deletions.
6 changes: 3 additions & 3 deletions etc/scripts/check_thirdparty.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
@click.command()
@click.option(
"-d",
"--dest_dir",
"--dest",
type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
required=True,
help="Path to the thirdparty directory to check.",
Expand All @@ -35,7 +35,7 @@
)
@click.help_option("-h", "--help")
def check_thirdparty_dir(
dest_dir,
dest,
wheels,
sdists,
):
Expand All @@ -45,7 +45,7 @@ def check_thirdparty_dir(
# check for problems
print(f"==> CHECK FOR PROBLEMS")
utils_thirdparty.find_problems(
dest_dir=dest_dir,
dest_dir=dest,
report_missing_sources=sdists,
report_missing_wheels=wheels,
)
Expand Down
191 changes: 80 additions & 111 deletions etc/scripts/fetch_thirdparty.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
import utils_thirdparty
import utils_requirements

TRACE = True
TRACE = False
TRACE_DEEP = False


@click.command()
Expand Down Expand Up @@ -99,11 +100,17 @@
"index_urls",
type=str,
metavar="INDEX",
default=utils_thirdparty.PYPI_INDEXES,
default=utils_thirdparty.PYPI_INDEX_URLS,
show_default=True,
multiple=True,
help="PyPI index URL(s) to use for wheels and sources, in order of preferences.",
)
@click.option(
"--use-cached-index",
is_flag=True,
help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.",
)

@click.help_option("-h", "--help")
def fetch_thirdparty(
requirements_files,
Expand All @@ -115,26 +122,34 @@ def fetch_thirdparty(
wheels,
sdists,
index_urls,
use_cached_index,
):
"""
Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions,
Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions,
and their ABOUT metadata, license and notices files.
Download the PyPI packages listed in the combination of:
- the pip requirements --requirements REQUIREMENT-FILE(s),
- the pip name==version --specifier SPECIFIER(s)
- any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR.
Download wheels with the --wheels option for the ``--python-version`` PYVER(s)
and ``--operating_system`` OS(s) combinations defaulting to all supported combinations.
Download wheels with the --wheels option for the ``--python-version``
PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all
supported combinations.
Download sdists tarballs with the --sdists option.
Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched.
Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels
and sources fetched.
Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs.
Download from the provided PyPI simple --index-url INDEX(s) URLs.
"""
if not (wheels or sdists):
print("Error: one or both of --wheels and --sdists is required.")
sys.exit(1)

print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}")

existing_packages_by_nv = {
(package.name, package.version): package
for package in utils_thirdparty.get_local_packages(directory=dest_dir)
Expand All @@ -150,134 +165,88 @@ def fetch_thirdparty(
required_name_versions.update(nvs)

for specifier in specifiers:
nv = utils_requirements.get_name_version(
nv = utils_requirements.get_required_name_version(
requirement=specifier,
with_unpinned=latest_version,
)
required_name_versions.add(nv)

if latest_version:
names = set(name for name, _version in sorted(required_name_versions))
required_name_versions = {(n, None) for n in names}

if not required_name_versions:
print("Error: no requirements requested.")
sys.exit(1)

if not os.listdir(dest_dir) and not (wheels or sdists):
print("Error: one or both of --wheels and --sdists is required.")
sys.exit(1)

if latest_version:
latest_name_versions = set()
names = set(name for name, _version in sorted(required_name_versions))
for name in sorted(names):
latests = utils_thirdparty.PypiPackage.sorted(
utils_thirdparty.get_package_versions(
name=name, version=None, index_urls=index_urls
)
)
if not latests:
print(f"No distribution found for: {name}")
continue
latest = latests[-1]
latest_name_versions.add((latest.name, latest.version))
required_name_versions = latest_name_versions

if TRACE:
print("required_name_versions:", required_name_versions)
if TRACE_DEEP:
print("required_name_versions:")
for n, v in required_name_versions:
print(f" {n} @ {v}")

# create the environments matrix we need for wheels
environments = None
if wheels:
# create the environments matrix we need for wheels
evts = itertools.product(python_versions, operating_systems)
environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts]

wheels_not_found = {}
sdists_not_found = {}
# iterate over requirements, one at a time
# Collect PyPI repos
repos = []
for index_url in index_urls:
index_url = index_url.strip("/")
existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url)
if existing:
existing.use_cached_index = use_cached_index
repos.append(existing)
else:
repo = utils_thirdparty.PypiSimpleRepository(
index_url=index_url,
use_cached_index=use_cached_index,
)
repos.append(repo)

wheels_fetched = []
wheels_not_found = []

sdists_fetched = []
sdists_not_found = []

for name, version in sorted(required_name_versions):
nv = name, version
existing_package = existing_packages_by_nv.get(nv)
print(f"Processing: {name} @ {version}")
if wheels:
for environment in environments:
if existing_package:
existing_wheels = list(
existing_package.get_supported_wheels(environment=environment)
)
else:
existing_wheels = None

if existing_wheels:
if TRACE:
print(
f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}"
)
if all(w.is_pure() for w in existing_wheels):
break
else:
continue

if TRACE:
print(f"Fetching wheel for: {name}=={version} on: {environment}")

try:
(
fetched_wheel_filenames,
existing_wheel_filenames,
) = utils_thirdparty.download_wheel(
name=name,
version=version,
environment=environment,
dest_dir=dest_dir,
index_urls=index_urls,
)
if TRACE:
if existing_wheel_filenames:
print(
f" ====> Wheels already available: {name}=={version} on: {environment}"
)
for whl in existing_wheel_filenames:
print(f" {whl}")
if fetched_wheel_filenames:
print(f" ====> Wheels fetched: {name}=={version} on: {environment}")
for whl in fetched_wheel_filenames:
print(f" {whl}")

fwfns = fetched_wheel_filenames + existing_wheel_filenames

if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns):
break

except utils_thirdparty.DistributionNotFound as e:
wheels_not_found[f"{name}=={version}"] = str(e)

if sdists:
if existing_package and existing_package.sdist:
if TRACE:
print(
f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}"
)
continue

if TRACE:
print(f" Fetching sdist for: {name}=={version}")

try:
fetched = utils_thirdparty.download_sdist(
print(f" ==> Fetching wheel for envt: {environment}")
fwfns = utils_thirdparty.download_wheel(
name=name,
version=version,
environment=environment,
dest_dir=dest_dir,
index_urls=index_urls,
repos=repos,
)
if fwfns:
wheels_fetched.extend(fwfns)
else:
wheels_not_found.append(f"{name}=={version} for: {environment}")
if TRACE:
print(f" NOT FOUND")

if sdists:
if TRACE:
print(f" ==> Fetching sdist: {name}=={version}")
fetched = utils_thirdparty.download_sdist(
name=name,
version=version,
dest_dir=dest_dir,
repos=repos,
)
if fetched:
sdists_fetched.append(fetched)
else:
sdists_not_found.append(f"{name}=={version}")
if TRACE:
if not fetched:
print(
f" ====> Sdist already available: {name}=={version}"
)
else:
print(
f" ====> Sdist fetched: {fetched} for {name}=={version}"
)

except utils_thirdparty.DistributionNotFound as e:
sdists_not_found[f"{name}=={version}"] = str(e)
print(f" NOT FOUND")

if wheels and wheels_not_found:
print(f"==> MISSING WHEELS")
Expand All @@ -290,7 +259,7 @@ def fetch_thirdparty(
print(f" {sd}")

print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES")
utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir)
utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index)
utils_thirdparty.clean_about_files(dest_dir=dest_dir)

# check for problems
Expand Down
22 changes: 11 additions & 11 deletions etc/scripts/gen_pypi_simple.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,26 +25,26 @@ class InvalidDistributionFilename(Exception):

def get_package_name_from_filename(filename):
"""
Return the package name extracted from a package ``filename``.
Optionally ``normalize`` the name according to distribution name rules.
Return the normalized package name extracted from a package ``filename``.
Normalization is done according to distribution name rules.
Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid::
>>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz")
'foo'
>>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl")
>>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl")
'foo-bar'
>>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl")
'cython'
>>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl")
'python-ldap'
>>> get_package_name_from_filename("foo.whl")
Traceback (most recent call last):
...
InvalidDistributionFilename: ...
>>> get_package_name_from_filename("foo.png")
Traceback (most recent call last):
...
InvalidFilePackageName: ...
>>> try:
... get_package_name_from_filename("foo.whl")
... except InvalidDistributionFilename:
... pass
>>> try:
... get_package_name_from_filename("foo.png")
... except InvalidDistributionFilename:
... pass
"""
if not filename or not filename.endswith(dist_exts):
raise InvalidDistributionFilename(filename)
Expand Down
5 changes: 2 additions & 3 deletions etc/scripts/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
aboutcode_toolkit
github-release-retry2
attrs
commoncode
click
requests
saneyaml
romp
pip
setuptools
twine
wheel
wheel
build
18 changes: 10 additions & 8 deletions etc/scripts/utils_requirements.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,23 +41,23 @@ def get_required_name_versions(requirement_lines, with_unpinned=False):
if req_line.startswith("-") or (not with_unpinned and not "==" in req_line):
print(f"Requirement line is not supported: ignored: {req_line}")
continue
yield get_name_version(requirement=req_line, with_unpinned=with_unpinned)
yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned)


def get_name_version(requirement, with_unpinned=False):
def get_required_name_version(requirement, with_unpinned=False):
"""
Return a (name, version) tuple given a`requirement` specifier string.
Requirement version must be pinned. If ``with_unpinned`` is True, unpinned
requirements are accepted and only the name portion is returned.
For example:
>>> assert get_name_version("foo==1.2.3") == ("foo", "1.2.3")
>>> assert get_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1")
>>> assert get_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3")
>>> assert get_name_version("foo", with_unpinned=True) == ("foo", "")
>>> assert get_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_name_version("foo>=1.2")
>>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3")
>>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1")
>>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3")
>>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "")
>>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2")
>>> try:
... assert not get_name_version("foo", with_unpinned=False)
... assert not get_required_name_version("foo", with_unpinned=False)
... except Exception as e:
... assert "Requirement version must be pinned" in str(e)
"""
Expand Down Expand Up @@ -110,6 +110,8 @@ def get_installed_reqs(site_packages_dir):
Return the installed pip requirements as text found in `site_packages_dir`
as a text.
"""
if not os.path.exists(site_packages_dir):
raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists")
# Also include these packages in the output with --all: wheel, distribute,
# setuptools, pip
args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir]
Expand Down
Loading

0 comments on commit 69bfc1b

Please sign in to comment.