From 421d1751d0a1883c387e4b0bec7167053346834c Mon Sep 17 00:00:00 2001 From: No Author Date: Fri, 28 Feb 2003 18:13:00 +0000 Subject: [PATCH 01/62] New repository initialized by cvs2svn. git-svn-id: svn+ssh://rubyforge.org/var/svn/rubygems/trunk@1 3d4018f9-ac1a-0410-99e9-8a154d859a19 From 566451360f78962ec0f74706d7a9012505f9a2e6 Mon Sep 17 00:00:00 2001 From: agustinhenze Date: Wed, 31 Jul 2013 04:08:08 -0700 Subject: [PATCH 02/62] Initial commit From c7ba5bd8afd74d04e82183d07f829439d2c74808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Suszy=C5=84ski=20Krzysztof?= Date: Tue, 13 Jun 2017 16:51:12 +0200 Subject: [PATCH 03/62] Initial commit From c83207d2c4f7b1fdc1e42d8af2756e73e953d312 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 18 Jan 2020 13:08:36 +0100 Subject: [PATCH 04/62] Initial commit From b46d84f6ae633105ce0b1ff51714e34778d608f5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 11 Oct 2021 22:29:48 +0200 Subject: [PATCH 05/62] Handle as_text correctly in cache Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 40 ++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) mode change 100644 => 100755 etc/scripts/utils_thirdparty.py diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py old mode 100644 new mode 100755 index d77afc3..99b9c0e --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -172,11 +172,20 @@ def fetch_wheels( else: force_pinned = False - rrp = list(get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - )) + try: + rrp = list(get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + )) + except Exception as e: + raise Exception( + dict( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + ) + ) from e fetched_filenames = set() for name, version, package in rrp: @@ -211,6 +220,7 @@ def fetch_wheels( print(f'Missed package {nv} in remote repo, has only:') for pv in rr.get_versions(n): print(' ', pv) + raise Exception('Missed some packages in remote repo') def fetch_sources( @@ -261,6 +271,8 @@ def fetch_sources( fetched = package.fetch_sdist(dest_dir=dest_dir) error = f'Failed to fetch' if not fetched else None yield package, error + if missed: + raise Exception(f'Missing source packages in {remote_links_url}', missed) ################################################################################ # @@ -693,8 +705,7 @@ def save_if_modified(location, content): return False if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}') - wmode = 'wb' if isinstance(content, bytes) else 'w' - with open(location, wmode, encoding="utf-8") as fo: + with open(location, 'w') as fo: fo.write(content) return True @@ -1845,7 +1856,7 @@ def get(self, path_or_url, as_text=True): if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = 'w' if as_text else 'wb' - with open(cached, wmode, encoding="utf-8") as fo: + with open(cached, wmode) as fo: fo.write(content) return content else: @@ -1857,7 +1868,7 @@ def put(self, filename, content): """ cached = os.path.join(self.directory, filename) wmode = 'wb' if isinstance(content, bytes) else 'w' - with open(cached, wmode, encoding="utf-8") as fo: + with open(cached, wmode) as fo: fo.write(content) @@ -2331,7 +2342,7 @@ def get_required_remote_packages( repo = get_remote_repo(remote_links_url=remote_links_url) else: # a local path - assert os.path.exists(remote_links_url) + assert os.path.exists(remote_links_url), f'Path does not exist: {remote_links_url}' repo = get_local_repo(directory=remote_links_url) for name, version in required_name_versions: @@ -2365,7 +2376,7 @@ def update_requirements(name, version=None, requirements_file='requirements.txt' updated_name_versions = sorted(updated_name_versions) nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions) - with open(requirements_file, 'w', encoding="utf-8") as fo: + with open(requirements_file, 'w') as fo: fo.write(nvs) @@ -2383,7 +2394,7 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.t raise Exception(f'Missing required package {name}=={version}') hashed.append(package.specifier_with_hashes) - with open(requirements_file, 'w', encoding="utf-8") as fo: + with open(requirements_file, 'w') as fo: fo.write('\n'.join(hashed)) ################################################################################ @@ -2915,7 +2926,7 @@ def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f'venv/bin/about check {dest_dir}'.split()) + subprocess.check_output(f'about check {dest_dir}'.split()) except subprocess.CalledProcessError as cpe: print() print('Invalid ABOUT files:') @@ -2953,7 +2964,6 @@ def find_problems( check_about(dest_dir=dest_dir) - def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return @@ -2962,4 +2972,4 @@ def compute_normalized_license_expression(declared_licenses): return pypi.compute_normalized_license(declared_licenses) except ImportError: # Scancode is not installed, we join all license strings and return it - return ' '.join(declared_licenses) + return ' '.join(declared_licenses).lower() From 255a898ceb3c440ad38c38d08dab7e2e9463771b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 11 Oct 2021 22:29:48 +0200 Subject: [PATCH 06/62] Handle as_text correctly in cache Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 1 - 1 file changed, 1 deletion(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 7613a0c..6b268ca 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -2965,7 +2965,6 @@ def find_problems( check_about(dest_dir=dest_dir) - def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return From e5833d13d8493af3ee385c63b76d4bd55aacbe15 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 18 Oct 2021 13:53:58 +0200 Subject: [PATCH 07/62] Add support for Python 3.10 Signed-off-by: Philippe Ombredanne --- etc/scripts/README.rst | 12 ++++-------- etc/scripts/utils_thirdparty.py | 7 ++++++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index 4cb6ec7..d8b00f9 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -1,10 +1,6 @@ -This directory contains the tools to: - -- manage a directory of thirdparty Python package source, wheels and metadata: - pin, build, update, document and publish to a PyPI-like repo (GitHub release) - -- build and publish scancode releases as wheel, sources and OS-specific bundles. - +This directory contains the tools to manage a directory of thirdparty Python +package source, wheels and metadata pin, build, update, document and publish to +a PyPI-like repo (GitHub release). NOTE: These are tested to run ONLY on Linux. @@ -38,7 +34,7 @@ Scripts ~~~~~~~ **gen_requirements.py**: create/update requirements files from currently - installed requirements. + installed requirements. **gen_requirements_dev.py** does the same but can subtract the main requirements to get extra requirements used in only development. diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 5cac536..444b20d 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -87,13 +87,14 @@ TRACE = False # Supported environments -PYTHON_VERSIONS = '36', '37', '38', '39', +PYTHON_VERSIONS = '36', '37', '38', '39', '310' ABIS_BY_PYTHON_VERSION = { '36':['cp36', 'cp36m'], '37':['cp37', 'cp37m'], '38':['cp38', 'cp38m'], '39':['cp39', 'cp39m'], + '310':['cp310', 'cp310m'], } PLATFORMS_BY_OS = { @@ -102,6 +103,7 @@ 'manylinux1_x86_64', 'manylinux2014_x86_64', 'manylinux2010_x86_64', + 'manylinux_2_12_x86_64', ], 'macos': [ 'macosx_10_6_intel', 'macosx_10_6_x86_64', @@ -112,6 +114,9 @@ 'macosx_10_13_intel', 'macosx_10_13_x86_64', 'macosx_10_14_intel', 'macosx_10_14_x86_64', 'macosx_10_15_intel', 'macosx_10_15_x86_64', + 'macosx_10_15_x86_64', + 'macosx_11_0_x86_64', + # 'macosx_11_0_arm64', ], 'windows': [ 'win_amd64', From 0a0ef125bfad078529070e7ee0e6caf6af70b331 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 17:37:15 +0200 Subject: [PATCH 08/62] Adopt black style Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 23 +- etc/scripts/bootstrap.py | 122 ++- etc/scripts/build_wheels.py | 60 +- etc/scripts/check_thirdparty.py | 11 +- etc/scripts/fetch_requirements.py | 80 +- etc/scripts/fix_thirdparty.py | 38 +- etc/scripts/gen_pypi_simple.py | 20 +- etc/scripts/gen_requirements.py | 21 +- etc/scripts/gen_requirements_dev.py | 35 +- etc/scripts/publish_files.py | 90 +- .../test_utils_pip_compatibility_tags.py | 70 +- etc/scripts/test_utils_pypi_supported_tags.py | 1 + etc/scripts/utils_dejacode.py | 78 +- etc/scripts/utils_pip_compatibility_tags.py | 28 +- etc/scripts/utils_pypi_supported_tags.py | 6 +- etc/scripts/utils_requirements.py | 28 +- etc/scripts/utils_thirdparty.py | 960 ++++++++++-------- 17 files changed, 930 insertions(+), 741 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 529cae3..b792d9f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,9 +17,9 @@ # -- Project information ----------------------------------------------------- -project = 'nexb-skeleton' -copyright = 'nexb Inc.' -author = 'nexb Inc.' +project = "nexb-skeleton" +copyright = "nexb Inc." +author = "nexb Inc." # -- General configuration --------------------------------------------------- @@ -27,11 +27,10 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [ -] +extensions = [] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -44,20 +43,20 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] html_context = { - 'css_files': [ - '_static/theme_overrides.css', # override wide tables in RTD theme - ], + "css_files": [ + "_static/theme_overrides.css", # override wide tables in RTD theme + ], "display_github": True, "github_user": "nexB", "github_repo": "nexb-skeleton", "github_version": "develop", # branch "conf_py_path": "/docs/source/", # path in the checkout to the docs root - } \ No newline at end of file +} diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py index fde505b..31f2f55 100644 --- a/etc/scripts/bootstrap.py +++ b/etc/scripts/bootstrap.py @@ -19,52 +19,63 @@ @click.command() - -@click.option('-r', '--requirements-file', +@click.option( + "-r", + "--requirements-file", type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar='FILE', + metavar="FILE", multiple=True, - default=['requirements.txt'], + default=["requirements.txt"], show_default=True, - help='Path to the requirements file(s) to use for thirdparty packages.', + help="Path to the requirements file(s) to use for thirdparty packages.", ) -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar='DIR', + metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, - help='Path to the thirdparty directory where wheels are built and ' - 'sources, ABOUT and LICENSE files fetched.', + help="Path to the thirdparty directory where wheels are built and " + "sources, ABOUT and LICENSE files fetched.", ) -@click.option('-p', '--python-version', +@click.option( + "-p", + "--python-version", type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar='PYVER', + metavar="PYVER", default=utils_thirdparty.PYTHON_VERSIONS, show_default=True, multiple=True, - help='Python version(s) to use for this build.', + help="Python version(s) to use for this build.", ) -@click.option('-o', '--operating-system', +@click.option( + "-o", + "--operating-system", type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar='OS', + metavar="OS", default=tuple(utils_thirdparty.PLATFORMS_BY_OS), multiple=True, show_default=True, - help='OS(ses) to use for this build: one of linux, mac or windows.', + help="OS(ses) to use for this build: one of linux, mac or windows.", ) -@click.option('-l', '--latest-version', +@click.option( + "-l", + "--latest-version", is_flag=True, - help='Get the latest version of all packages, ignoring version specifiers.', + help="Get the latest version of all packages, ignoring version specifiers.", ) -@click.option('--sync-dejacode', +@click.option( + "--sync-dejacode", is_flag=True, - help='Synchronize packages with DejaCode.', + help="Synchronize packages with DejaCode.", ) -@click.option('--with-deps', +@click.option( + "--with-deps", is_flag=True, - help='Also include all dependent wheels.', + help="Also include all dependent wheels.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def bootstrap( requirements_file, thirdparty_dir, @@ -105,18 +116,19 @@ def bootstrap( required_name_versions = set() for req_file in requirements_files: - nvs = utils_thirdparty.load_requirements( - requirements_file=req_file, force_pinned=False) + nvs = utils_thirdparty.load_requirements(requirements_file=req_file, force_pinned=False) required_name_versions.update(nvs) if latest_version: required_name_versions = set((name, None) for name, _ver in required_name_versions) - print(f'PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES') + print( + f"PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES" + ) # fetch all available wheels, keep track of missing # start with local, then remote, then PyPI - print('==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS') + print("==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS") # list of all the wheel filenames either pre-existing, fetched or built # updated as we progress available_wheel_filenames = [] @@ -131,19 +143,32 @@ def bootstrap( # start with a local check for (name, version), envt in itertools.product(required_name_versions, environments): - local_pack = local_packages_by_namever.get((name, version,)) + local_pack = local_packages_by_namever.get( + ( + name, + version, + ) + ) if local_pack: supported_wheels = list(local_pack.get_supported_wheels(environment=envt)) if supported_wheels: available_wheel_filenames.extend(w.filename for w in supported_wheels) - print(f'====> No fetch or build needed. ' - f'Local wheel already available for {name}=={version} ' - f'on os: {envt.operating_system} for Python: {envt.python_version}') + print( + f"====> No fetch or build needed. " + f"Local wheel already available for {name}=={version} " + f"on os: {envt.operating_system} for Python: {envt.python_version}" + ) continue - name_version_envt_to_fetch.append((name, version, envt,)) + name_version_envt_to_fetch.append( + ( + name, + version, + envt, + ) + ) - print(f'==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS') + print(f"==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS") # list of (name, version, environment) not fetch and to build name_version_envt_to_build = [] @@ -161,46 +186,53 @@ def bootstrap( if fetched_fwn: available_wheel_filenames.append(fetched_fwn) else: - name_version_envt_to_build.append((name, version, envt,)) + name_version_envt_to_build.append( + ( + name, + version, + envt, + ) + ) # At this stage we have all the wheels we could obtain without building for name, version, envt in name_version_envt_to_build: - print(f'====> Need to build wheels for {name}=={version} on os: ' - f'{envt.operating_system} for Python: {envt.python_version}') + print( + f"====> Need to build wheels for {name}=={version} on os: " + f"{envt.operating_system} for Python: {envt.python_version}" + ) packages_and_envts_to_build = [ - (PypiPackage(name, version), envt) - for name, version, envt in name_version_envt_to_build + (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build ] - print(f'==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS') + print(f"==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS") package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( packages_and_envts=packages_and_envts_to_build, build_remotely=build_remotely, with_deps=with_deps, dest_dir=thirdparty_dir, -) + ) if wheel_filenames_built: available_wheel_filenames.extend(available_wheel_filenames) for pack, envt in package_envts_not_built: print( - f'====> FAILED to build any wheel for {pack.name}=={pack.version} ' - f'on os: {envt.operating_system} for Python: {envt.python_version}' + f"====> FAILED to build any wheel for {pack.name}=={pack.version} " + f"on os: {envt.operating_system} for Python: {envt.python_version}" ) - print(f'==> FETCHING SOURCE DISTRIBUTIONS') + print(f"==> FETCHING SOURCE DISTRIBUTIONS") # fetch all sources, keep track of missing # This is a list of (name, version) utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - print(f'==> FETCHING ABOUT AND LICENSE FILES') + print(f"==> FETCHING ABOUT AND LICENSE FILES") utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) ############################################################################ if sync_dejacode: - print(f'==> SYNC WITH DEJACODE') + print(f"==> SYNC WITH DEJACODE") # try to fetch from DejaCode any missing ABOUT # create all missing DejaCode packages pass @@ -208,5 +240,5 @@ def bootstrap( utils_thirdparty.find_problems(dest_dir=thirdparty_dir) -if __name__ == '__main__': +if __name__ == "__main__": bootstrap() diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py index 352b705..5a39c78 100644 --- a/etc/scripts/build_wheels.py +++ b/etc/scripts/build_wheels.py @@ -14,55 +14,67 @@ @click.command() - -@click.option('-n', '--name', +@click.option( + "-n", + "--name", type=str, - metavar='PACKAGE_NAME', + metavar="PACKAGE_NAME", required=True, - help='Python package name to add or build.', + help="Python package name to add or build.", ) -@click.option('-v', '--version', +@click.option( + "-v", + "--version", type=str, default=None, - metavar='VERSION', - help='Python package version to add or build.', + metavar="VERSION", + help="Python package version to add or build.", ) -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar='DIR', + metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, - help='Path to the thirdparty directory where wheels are built.', + help="Path to the thirdparty directory where wheels are built.", ) -@click.option('-p', '--python-version', +@click.option( + "-p", + "--python-version", type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar='PYVER', + metavar="PYVER", default=utils_thirdparty.PYTHON_VERSIONS, show_default=True, multiple=True, - help='Python version to use for this build.', + help="Python version to use for this build.", ) -@click.option('-o', '--operating-system', +@click.option( + "-o", + "--operating-system", type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar='OS', + metavar="OS", default=tuple(utils_thirdparty.PLATFORMS_BY_OS), multiple=True, show_default=True, - help='OS to use for this build: one of linux, mac or windows.', + help="OS to use for this build: one of linux, mac or windows.", ) -@click.option('--build-remotely', +@click.option( + "--build-remotely", is_flag=True, - help='Build missing wheels remotely.', + help="Build missing wheels remotely.", ) -@click.option('--with-deps', +@click.option( + "--with-deps", is_flag=True, - help='Also include all dependent wheels.', + help="Also include all dependent wheels.", ) -@click.option('--verbose', +@click.option( + "--verbose", is_flag=True, - help='Provide verbose output.', + help="Provide verbose output.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def build_wheels( name, version, @@ -93,5 +105,5 @@ def build_wheels( ) -if __name__ == '__main__': +if __name__ == "__main__": build_wheels() diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index e48cfce..4fea16c 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -14,13 +14,14 @@ @click.command() - -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, - help='Path to the thirdparty directory to check.', + help="Path to the thirdparty directory to check.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def check_thirdparty_dir(thirdparty_dir): """ Check a thirdparty directory for problems. @@ -28,5 +29,5 @@ def check_thirdparty_dir(thirdparty_dir): utils_thirdparty.find_problems(dest_dir=thirdparty_dir) -if __name__ == '__main__': +if __name__ == "__main__": check_thirdparty_dir() diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py index 21de865..9da9ce9 100644 --- a/etc/scripts/fetch_requirements.py +++ b/etc/scripts/fetch_requirements.py @@ -16,64 +16,78 @@ @click.command() - -@click.option('-r', '--requirements-file', +@click.option( + "-r", + "--requirements-file", type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar='FILE', + metavar="FILE", multiple=True, - default=['requirements.txt'], + default=["requirements.txt"], show_default=True, - help='Path to the requirements file to use for thirdparty packages.', + help="Path to the requirements file to use for thirdparty packages.", ) -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar='DIR', + metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, - help='Path to the thirdparty directory.', + help="Path to the thirdparty directory.", ) -@click.option('-p', '--python-version', +@click.option( + "-p", + "--python-version", type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar='INT', + metavar="INT", multiple=True, - default=['36'], + default=["36"], show_default=True, - help='Python version to use for this build.', + help="Python version to use for this build.", ) -@click.option('-o', '--operating-system', +@click.option( + "-o", + "--operating-system", type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar='OS', + metavar="OS", multiple=True, - default=['linux'], + default=["linux"], show_default=True, - help='OS to use for this build: one of linux, mac or windows.', + help="OS to use for this build: one of linux, mac or windows.", ) -@click.option('-s', '--with-sources', +@click.option( + "-s", + "--with-sources", is_flag=True, - help='Fetch the corresponding source distributions.', + help="Fetch the corresponding source distributions.", ) -@click.option('-a', '--with-about', +@click.option( + "-a", + "--with-about", is_flag=True, - help='Fetch the corresponding ABOUT and LICENSE files.', + help="Fetch the corresponding ABOUT and LICENSE files.", ) -@click.option('--allow-unpinned', +@click.option( + "--allow-unpinned", is_flag=True, - help='Allow requirements without pinned versions.', + help="Allow requirements without pinned versions.", ) -@click.option('-s', '--only-sources', +@click.option( + "-s", + "--only-sources", is_flag=True, - help='Fetch only the corresponding source distributions.', + help="Fetch only the corresponding source distributions.", ) -@click.option('-u', '--remote-links-url', +@click.option( + "-u", + "--remote-links-url", type=str, - metavar='URL', + metavar="URL", default=utils_thirdparty.REMOTE_LINKS_URL, show_default=True, - help='URL to a PyPI-like links web site. ' - 'Or local path to a directory with wheels.', + help="URL to a PyPI-like links web site. " "Or local path to a directory with wheels.", ) - -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def fetch_requirements( requirements_file, thirdparty_dir, @@ -117,7 +131,7 @@ def fetch_requirements( remote_links_url=remote_links_url, ): if error: - print('Failed to fetch wheel:', package, ':', error) + print("Failed to fetch wheel:", package, ":", error) # optionally fetch sources if with_sources or only_sources: @@ -130,7 +144,7 @@ def fetch_requirements( remote_links_url=remote_links_url, ): if error: - print('Failed to fetch source:', package, ':', error) + print("Failed to fetch source:", package, ":", error) if with_about: utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) @@ -141,5 +155,5 @@ def fetch_requirements( ) -if __name__ == '__main__': +if __name__ == "__main__": fetch_requirements() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py index 061d3fa..9b1cbc4 100644 --- a/etc/scripts/fix_thirdparty.py +++ b/etc/scripts/fix_thirdparty.py @@ -14,21 +14,24 @@ @click.command() - -@click.option('-d', '--thirdparty-dir', +@click.option( + "-d", + "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, - help='Path to the thirdparty directory to fix.', + help="Path to the thirdparty directory to fix.", ) -@click.option('--build-wheels', +@click.option( + "--build-wheels", is_flag=True, - help='Build all missing wheels .', + help="Build all missing wheels .", ) -@click.option('--build-remotely', +@click.option( + "--build-remotely", is_flag=True, - help='Build missing wheels remotely.', + help="Build missing wheels remotely.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def fix_thirdparty_dir( thirdparty_dir, build_wheels, @@ -47,35 +50,36 @@ def fix_thirdparty_dir( Optionally build missing binary wheels for all supported OS and Python version combos locally or remotely. """ - print('***FETCH*** MISSING WHEELS') + print("***FETCH*** MISSING WHEELS") package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) - print('***FETCH*** MISSING SOURCES') + print("***FETCH*** MISSING SOURCES") src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) package_envts_not_built = [] if build_wheels: - print('***BUILD*** MISSING WHEELS') + print("***BUILD*** MISSING WHEELS") package_envts_not_built, _wheel_filenames_built = utils_thirdparty.build_missing_wheels( packages_and_envts=package_envts_not_fetched, build_remotely=build_remotely, dest_dir=thirdparty_dir, ) - print('***ADD*** ABOUT AND LICENSES') + print("***ADD*** ABOUT AND LICENSES") utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) # report issues for name, version in src_name_ver_not_fetched: - print(f'{name}=={version}: Failed to fetch source distribution.') + print(f"{name}=={version}: Failed to fetch source distribution.") for package, envt in package_envts_not_built: print( - f'{package.name}=={package.version}: Failed to build wheel ' - f'on {envt.operating_system} for Python {envt.python_version}') + f"{package.name}=={package.version}: Failed to build wheel " + f"on {envt.operating_system} for Python {envt.python_version}" + ) - print('***FIND PROBLEMS***') + print("***FIND PROBLEMS***") utils_thirdparty.find_problems(dest_dir=thirdparty_dir) -if __name__ == '__main__': +if __name__ == "__main__": fix_thirdparty_dir() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 887e407..53db9b0 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -45,10 +45,15 @@ r"""^(?P(?P.+?)-(?P.*?)) ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) \.whl)$""", - re.VERBOSE + re.VERBOSE, ).match -sdist_exts = ".tar.gz", ".tar.bz2", ".zip", ".tar.xz", +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) wheel_ext = ".whl" app_ext = ".pyz" dist_exts = sdist_exts + (wheel_ext, app_ext) @@ -98,7 +103,7 @@ def get_package_name_from_filename(filename, normalize=True): if not extension or not name_ver: raise InvalidDistributionFilename(filename) - name, _, version = name_ver.rpartition('-') + name, _, version = name_ver.rpartition("-") if not (name and version): raise InvalidDistributionFilename(filename) @@ -110,8 +115,8 @@ def get_package_name_from_filename(filename, normalize=True): if not wheel_info: raise InvalidDistributionFilename(filename) - name = wheel_info.group('name') - version = wheel_info.group('version') + name = wheel_info.group("name") + version = wheel_info.group("version") if not (name and version): raise InvalidDistributionFilename(filename) @@ -120,7 +125,7 @@ def get_package_name_from_filename(filename, normalize=True): name_ver, extension, _ = filename.rpartition(".pyz") if "-" in filename: - name, _, version = name_ver.rpartition('-') + name, _, version = name_ver.rpartition("-") else: name = name_ver @@ -128,7 +133,7 @@ def get_package_name_from_filename(filename, normalize=True): raise InvalidDistributionFilename(filename) if normalize: - name = name.lower().replace('_', '-') + name = name.lower().replace("_", "-") return name @@ -187,5 +192,6 @@ def build_pypi_index(directory, write_index=False): if __name__ == "__main__": import sys + pkg_dir = sys.argv[1] build_pypi_index(pkg_dir) diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 3be974c..6f17a75 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -13,21 +13,24 @@ @click.command() - -@click.option('-s', '--site-packages-dir', +@click.option( + "-s", + "--site-packages-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), required=True, - metavar='DIR', + metavar="DIR", help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', ) -@click.option('-r', '--requirements-file', +@click.option( + "-r", + "--requirements-file", type=click.Path(path_type=str, dir_okay=False), - metavar='FILE', - default='requirements.txt', + metavar="FILE", + default="requirements.txt", show_default=True, - help='Path to the requirements file to update or create.', + help="Path to the requirements file to update or create.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def gen_requirements(site_packages_dir, requirements_file): """ Create or replace the `--requirements-file` file FILE requirements file with all @@ -39,5 +42,5 @@ def gen_requirements(site_packages_dir, requirements_file): ) -if __name__ == '__main__': +if __name__ == "__main__": gen_requirements() diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index ff4ce50..ef80455 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -13,29 +13,34 @@ @click.command() - -@click.option('-s', '--site-packages-dir', +@click.option( + "-s", + "--site-packages-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), required=True, - metavar='DIR', + metavar="DIR", help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', ) -@click.option('-d', '--dev-requirements-file', +@click.option( + "-d", + "--dev-requirements-file", type=click.Path(path_type=str, dir_okay=False), - metavar='FILE', - default='requirements-dev.txt', + metavar="FILE", + default="requirements-dev.txt", show_default=True, - help='Path to the dev requirements file to update or create.', + help="Path to the dev requirements file to update or create.", ) -@click.option('-r', '--main-requirements-file', +@click.option( + "-r", + "--main-requirements-file", type=click.Path(path_type=str, dir_okay=False), - default='requirements.txt', - metavar='FILE', + default="requirements.txt", + metavar="FILE", show_default=True, - help='Path to the main requirements file. Its requirements will be excluded ' - 'from the generated dev requirements.', + help="Path to the main requirements file. Its requirements will be excluded " + "from the generated dev requirements.", ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirements_file): """ Create or overwrite the `--dev-requirements-file` pip requirements FILE with @@ -47,9 +52,9 @@ def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirem utils_requirements.lock_dev_requirements( dev_requirements_file=dev_requirements_file, main_requirements_file=main_requirements_file, - site_packages_dir=site_packages_dir + site_packages_dir=site_packages_dir, ) -if __name__ == '__main__': +if __name__ == "__main__": gen_dev_requirements() diff --git a/etc/scripts/publish_files.py b/etc/scripts/publish_files.py index f343cb3..8669363 100644 --- a/etc/scripts/publish_files.py +++ b/etc/scripts/publish_files.py @@ -32,7 +32,7 @@ def get_files(location): for top, _dirs, files in os.walk(location): for filename in files: pth = Path(os.path.join(top, filename)) - with open(pth, 'rb') as fi: + with open(pth, "rb") as fi: md5 = hashlib.md5(fi.read()).hexdigest() yield filename, pth, md5 @@ -43,20 +43,20 @@ def get_etag_md5(url): """ headers = utils_thirdparty.get_remote_headers(url) headers = {k.lower(): v for k, v in headers.items()} - etag = headers .get('etag') + etag = headers.get("etag") if etag: etag = etag.strip('"').lower() return etag def create_or_update_release_and_upload_directory( - user, - repo, - tag_name, - token, - directory, - retry_limit=10, - description=None, + user, + repo, + tag_name, + token, + directory, + retry_limit=10, + description=None, ): """ Create or update a GitHub release at https://github.com// for @@ -69,15 +69,16 @@ def create_or_update_release_and_upload_directory( Remote files that are not the same as the local files are deleted and re- uploaded. """ - release_homepage_url = f'https://github.com/{user}/{repo}/releases/{tag_name}' + release_homepage_url = f"https://github.com/{user}/{repo}/releases/{tag_name}" # scrape release page HTML for links - urls_by_filename = {os.path.basename(l): l + urls_by_filename = { + os.path.basename(l): l for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url) } # compute what is new, modified or unchanged - print(f'Compute which files is new, modified or unchanged in {release_homepage_url}') + print(f"Compute which files is new, modified or unchanged in {release_homepage_url}") new_to_upload = [] unchanged_to_skip = [] @@ -85,21 +86,21 @@ def create_or_update_release_and_upload_directory( for filename, pth, md5 in get_files(directory): url = urls_by_filename.get(filename) if not url: - print(f'{filename} content is NEW, will upload') + print(f"{filename} content is NEW, will upload") new_to_upload.append(pth) continue out_of_date = get_etag_md5(url) != md5 if out_of_date: - print(f'{url} content is CHANGED based on md5 etag, will re-upload') + print(f"{url} content is CHANGED based on md5 etag, will re-upload") modified_to_delete_and_reupload.append(pth) else: # print(f'{url} content is IDENTICAL, skipping upload based on Etag') unchanged_to_skip.append(pth) - print('.') + print(".") ghapi = grr.GithubApi( - github_api_url='https://api.github.com', + github_api_url="https://api.github.com", user=user, repo=repo, token=token, @@ -108,86 +109,89 @@ def create_or_update_release_and_upload_directory( # yank modified print( - f'Unpublishing {len(modified_to_delete_and_reupload)} published but ' - f'locally modified files in {release_homepage_url}') + f"Unpublishing {len(modified_to_delete_and_reupload)} published but " + f"locally modified files in {release_homepage_url}" + ) release = ghapi.get_release_by_tag(tag_name) for pth in modified_to_delete_and_reupload: filename = os.path.basename(pth) asset_id = ghapi.find_asset_id_by_file_name(filename, release) - print (f' Unpublishing file: {filename}).') + print(f" Unpublishing file: {filename}).") response = ghapi.delete_asset(asset_id) if response.status_code != requests.codes.no_content: # NOQA - raise Exception(f'failed asset deletion: {response}') + raise Exception(f"failed asset deletion: {response}") # finally upload new and modified to_upload = new_to_upload + modified_to_delete_and_reupload - print(f'Publishing with {len(to_upload)} files to {release_homepage_url}') + print(f"Publishing with {len(to_upload)} files to {release_homepage_url}") release = grr.Release(tag_name=tag_name, body=description) grr.make_release(ghapi, release, to_upload) TOKEN_HELP = ( - 'The Github personal acess token is used to authenticate API calls. ' - 'Required unless you set the GITHUB_TOKEN environment variable as an alternative. ' - 'See for details: https://github.com/settings/tokens and ' - 'https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token' + "The Github personal acess token is used to authenticate API calls. " + "Required unless you set the GITHUB_TOKEN environment variable as an alternative. " + "See for details: https://github.com/settings/tokens and " + "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token" ) @click.command() - @click.option( - '--user-repo-tag', - help='The GitHub qualified repository user/name/tag in which ' - 'to create the release such as in nexB/thirdparty/pypi', + "--user-repo-tag", + help="The GitHub qualified repository user/name/tag in which " + "to create the release such as in nexB/thirdparty/pypi", type=str, required=True, ) @click.option( - '-d', '--directory', - help='The directory that contains files to upload to the release.', + "-d", + "--directory", + help="The directory that contains files to upload to the release.", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), required=True, ) @click.option( - '--token', + "--token", help=TOKEN_HELP, - default=os.environ.get('GITHUB_TOKEN', None), + default=os.environ.get("GITHUB_TOKEN", None), type=str, required=False, ) @click.option( - '--description', - help='Text description for the release. Ignored if the release exists.', + "--description", + help="Text description for the release. Ignored if the release exists.", default=None, type=str, required=False, ) @click.option( - '--retry_limit', - help='Number of retries when making failing GitHub API calls. ' - 'Retrying helps work around transient failures of the GitHub API.', + "--retry_limit", + help="Number of retries when making failing GitHub API calls. " + "Retrying helps work around transient failures of the GitHub API.", type=int, default=10, ) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def publish_files( user_repo_tag, directory, - retry_limit=10, token=None, description=None, + retry_limit=10, + token=None, + description=None, ): """ Publish all the files in DIRECTORY as assets to a GitHub release. Either create or update/replace remote files' """ if not token: - click.secho('--token required option is missing.') + click.secho("--token required option is missing.") click.secho(TOKEN_HELP) sys.exit(1) - user, repo, tag_name = user_repo_tag.split('/') + user, repo, tag_name = user_repo_tag.split("/") create_or_update_release_and_upload_directory( user=user, @@ -200,5 +204,5 @@ def publish_files( ) -if __name__ == '__main__': +if __name__ == "__main__": publish_files() diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index 30c4dda..722fa70 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -33,23 +33,25 @@ import utils_pip_compatibility_tags -@pytest.mark.parametrize('version_info, expected', [ - ((2,), '2'), - ((2, 8), '28'), - ((3,), '3'), - ((3, 6), '36'), - # Test a tuple of length 3. - ((3, 6, 5), '36'), - # Test a 2-digit minor version. - ((3, 10), '310'), -]) +@pytest.mark.parametrize( + "version_info, expected", + [ + ((2,), "2"), + ((2, 8), "28"), + ((3,), "3"), + ((3, 6), "36"), + # Test a tuple of length 3. + ((3, 6, 5), "36"), + # Test a 2-digit minor version. + ((3, 10), "310"), + ], +) def test_version_info_to_nodot(version_info, expected): actual = pip_compatibility_tags.version_info_to_nodot(version_info) assert actual == expected class Testcompatibility_tags(object): - def mock_get_config_var(self, **kwd): """ Patch sysconfig.get_config_var for arbitrary keys. @@ -69,23 +71,25 @@ def test_no_hyphen_tag(self): """ import pip._internal.utils.compatibility_tags - mock_gcf = self.mock_get_config_var(SOABI='cpython-35m-darwin') + mock_gcf = self.mock_get_config_var(SOABI="cpython-35m-darwin") - with patch('sysconfig.get_config_var', mock_gcf): + with patch("sysconfig.get_config_var", mock_gcf): supported = pip._internal.utils.compatibility_tags.get_supported() for tag in supported: - assert '-' not in tag.interpreter - assert '-' not in tag.abi - assert '-' not in tag.platform + assert "-" not in tag.interpreter + assert "-" not in tag.abi + assert "-" not in tag.platform class TestManylinux2010Tags(object): - - @pytest.mark.parametrize("manylinux2010,manylinux1", [ - ("manylinux2010_x86_64", "manylinux1_x86_64"), - ("manylinux2010_i686", "manylinux1_i686"), - ]) + @pytest.mark.parametrize( + "manylinux2010,manylinux1", + [ + ("manylinux2010_x86_64", "manylinux1_x86_64"), + ("manylinux2010_i686", "manylinux1_i686"), + ], + ) def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): """ Specifying manylinux2010 implies manylinux1. @@ -93,22 +97,22 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): groups = {} supported = pip_compatibility_tags.get_supported(platforms=[manylinux2010]) for tag in supported: - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) for arches in groups.values(): - if arches == ['any']: + if arches == ["any"]: continue assert arches[:2] == [manylinux2010, manylinux1] class TestManylinux2014Tags(object): - - @pytest.mark.parametrize("manylinuxA,manylinuxB", [ - ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), - ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), - ]) + @pytest.mark.parametrize( + "manylinuxA,manylinuxB", + [ + ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), + ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), + ], + ) def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): """ Specifying manylinux2014 implies manylinux2010/manylinux1. @@ -116,13 +120,11 @@ def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): groups = {} supported = pip_compatibility_tags.get_supported(platforms=[manylinuxA]) for tag in supported: - groups.setdefault( - (tag.interpreter, tag.abi), [] - ).append(tag.platform) + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) expected_arches = [manylinuxA] expected_arches.extend(manylinuxB) for arches in groups.values(): - if arches == ['any']: + if arches == ["any"]: continue assert arches[:3] == expected_arches diff --git a/etc/scripts/test_utils_pypi_supported_tags.py b/etc/scripts/test_utils_pypi_supported_tags.py index 9ad68b2..d291572 100644 --- a/etc/scripts/test_utils_pypi_supported_tags.py +++ b/etc/scripts/test_utils_pypi_supported_tags.py @@ -29,6 +29,7 @@ def validate_wheel_filename_for_pypi(filename): an empty list if all tags are supported. """ from utils_thirdparty import Wheel + wheel = Wheel.from_filename(filename) return validate_platforms_for_pypi(wheel.platforms) diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index 8b6e5d2..f28e247 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -21,19 +21,19 @@ Utility to create and retrieve package and ABOUT file data from DejaCode. """ -DEJACODE_API_KEY = os.environ.get('DEJACODE_API_KEY', '') -DEJACODE_API_URL = os.environ.get('DEJACODE_API_URL', '') +DEJACODE_API_KEY = os.environ.get("DEJACODE_API_KEY", "") +DEJACODE_API_URL = os.environ.get("DEJACODE_API_URL", "") -DEJACODE_API_URL_PACKAGES = f'{DEJACODE_API_URL}packages/' +DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" DEJACODE_API_HEADERS = { - 'Authorization': 'Token {}'.format(DEJACODE_API_KEY), - 'Accept': 'application/json; indent=4', + "Authorization": "Token {}".format(DEJACODE_API_KEY), + "Accept": "application/json; indent=4", } def can_do_api_calls(): if not DEJACODE_API_KEY and DEJACODE_API_URL: - print('DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing') + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") return False else: return True @@ -53,7 +53,7 @@ def fetch_dejacode_packages(params): headers=DEJACODE_API_HEADERS, ) - return response.json()['results'] + return response.json()["results"] def get_package_data(distribution): @@ -68,9 +68,9 @@ def get_package_data(distribution): return results[0] elif len_results > 1: - print(f'More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}') + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") else: - print('Could not find package:', distribution.download_url) + print("Could not find package:", distribution.download_url) def update_with_dejacode_data(distribution): @@ -82,7 +82,7 @@ def update_with_dejacode_data(distribution): if package_data: return distribution.update(package_data, keep_extra=False) - print(f'No package found for: {distribution}') + print(f"No package found for: {distribution}") def update_with_dejacode_about_data(distribution): @@ -92,19 +92,19 @@ def update_with_dejacode_about_data(distribution): """ package_data = get_package_data(distribution) if package_data: - package_api_url = package_data['api_url'] - about_url = f'{package_api_url}about' + package_api_url = package_data["api_url"] + about_url = f"{package_api_url}about" response = requests.get(about_url, headers=DEJACODE_API_HEADERS) # note that this is YAML-formatted - about_text = response.json()['about_data'] + about_text = response.json()["about_data"] about_data = saneyaml.load(about_text) return distribution.update(about_data, keep_extra=True) - print(f'No package found for: {distribution}') + print(f"No package found for: {distribution}") -def fetch_and_save_about_files(distribution, dest_dir='thirdparty'): +def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): """ Fetch and save in `dest_dir` the .ABOUT, .LICENSE and .NOTICE files fetched from DejaCode for a Distribution `distribution`. Return True if files were @@ -112,8 +112,8 @@ def fetch_and_save_about_files(distribution, dest_dir='thirdparty'): """ package_data = get_package_data(distribution) if package_data: - package_api_url = package_data['api_url'] - about_url = f'{package_api_url}about_files' + package_api_url = package_data["api_url"] + about_url = f"{package_api_url}about_files" response = requests.get(about_url, headers=DEJACODE_API_HEADERS) about_zip = response.content with io.BytesIO(about_zip) as zf: @@ -121,7 +121,7 @@ def fetch_and_save_about_files(distribution, dest_dir='thirdparty'): zi.extractall(path=dest_dir) return True - print(f'No package found for: {distribution}') + print(f"No package found for: {distribution}") def find_latest_dejacode_package(distribution): @@ -138,9 +138,9 @@ def find_latest_dejacode_package(distribution): for package_data in packages: matched = ( - package_data['download_url'] == distribution.download_url - and package_data['version'] == distribution.version - and package_data['filename'] == distribution.filename + package_data["download_url"] == distribution.download_url + and package_data["version"] == distribution.version + and package_data["filename"] == distribution.filename ) if matched: @@ -149,12 +149,11 @@ def find_latest_dejacode_package(distribution): # there was no exact match, find the latest version # TODO: consider the closest version rather than the latest # or the version that has the best data - with_versions = [(packaging_version.parse(p['version']), p) for p in packages] + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( - f'Found DejaCode latest version: {latest_version} ' - f'for dist: {distribution.package_url}', + f"Found DejaCode latest version: {latest_version} " f"for dist: {distribution.package_url}", ) return latest_package_version @@ -172,27 +171,26 @@ def create_dejacode_package(distribution): if existing_package_data: return existing_package_data - print(f'Creating new DejaCode package for: {distribution}') + print(f"Creating new DejaCode package for: {distribution}") new_package_payload = { # Trigger data collection, scan, and purl - 'collect_data': 1, + "collect_data": 1, } fields_to_carry_over = [ - 'download_url' - 'type', - 'namespace', - 'name', - 'version', - 'qualifiers', - 'subpath', - 'license_expression', - 'copyright', - 'description', - 'homepage_url', - 'primary_language', - 'notice_text', + "download_url" "type", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "license_expression", + "copyright", + "description", + "homepage_url", + "primary_language", + "notice_text", ] for field in fields_to_carry_over: @@ -207,7 +205,7 @@ def create_dejacode_package(distribution): ) new_package_data = response.json() if response.status_code != 201: - raise Exception(f'Error, cannot create package for: {distribution}') + raise Exception(f"Error, cannot create package for: {distribution}") print(f'New Package created at: {new_package_data["absolute_url"]}') return new_package_data diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index 4c6529b..5d5eb34 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -36,13 +36,13 @@ mac_platforms, ) -_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') +_osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") def version_info_to_nodot(version_info): # type: (Tuple[int, ...]) -> str # Only use up to the first two numbers. - return ''.join(map(str, version_info[:2])) + return "".join(map(str, version_info[:2])) def _mac_platforms(arch): @@ -57,7 +57,7 @@ def _mac_platforms(arch): # actual prefix provided by the user in case they provided # something like "macosxcustom_". It may be good to remove # this as undocumented or deprecate it in the future. - '{}_{}'.format(name, arch[len('macosx_'):]) + "{}_{}".format(name, arch[len("macosx_") :]) for arch in mac_platforms(mac_version, actual_arch) ] else: @@ -69,31 +69,31 @@ def _mac_platforms(arch): def _custom_manylinux_platforms(arch): # type: (str) -> List[str] arches = [arch] - arch_prefix, arch_sep, arch_suffix = arch.partition('_') - if arch_prefix == 'manylinux2014': + arch_prefix, arch_sep, arch_suffix = arch.partition("_") + if arch_prefix == "manylinux2014": # manylinux1/manylinux2010 wheels run on most manylinux2014 systems # with the exception of wheels depending on ncurses. PEP 599 states # manylinux1/manylinux2010 wheels should be considered # manylinux2014 wheels: # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels - if arch_suffix in {'i686', 'x86_64'}: - arches.append('manylinux2010' + arch_sep + arch_suffix) - arches.append('manylinux1' + arch_sep + arch_suffix) - elif arch_prefix == 'manylinux2010': + if arch_suffix in {"i686", "x86_64"}: + arches.append("manylinux2010" + arch_sep + arch_suffix) + arches.append("manylinux1" + arch_sep + arch_suffix) + elif arch_prefix == "manylinux2010": # manylinux1 wheels run on most manylinux2010 systems with the # exception of wheels depending on ncurses. PEP 571 states # manylinux1 wheels should be considered manylinux2010 wheels: # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels - arches.append('manylinux1' + arch_sep + arch_suffix) + arches.append("manylinux1" + arch_sep + arch_suffix) return arches def _get_custom_platforms(arch): # type: (str) -> List[str] - arch_prefix, _arch_sep, _arch_suffix = arch.partition('_') - if arch.startswith('macosx'): + arch_prefix, _arch_sep, _arch_suffix = arch.partition("_") + if arch.startswith("macosx"): arches = _mac_platforms(arch) - elif arch_prefix in ['manylinux2014', 'manylinux2010']: + elif arch_prefix in ["manylinux2014", "manylinux2010"]: arches = _custom_manylinux_platforms(arch) else: arches = [arch] @@ -139,7 +139,7 @@ def get_supported( version=None, # type: Optional[str] platforms=None, # type: Optional[List[str]] impl=None, # type: Optional[str] - abis=None # type: Optional[List[str]] + abis=None, # type: Optional[List[str]] ): # type: (...) -> List[Tag] """Return a list of supported tags for each version specified in diff --git a/etc/scripts/utils_pypi_supported_tags.py b/etc/scripts/utils_pypi_supported_tags.py index 8dcb70f..de9f21b 100644 --- a/etc/scripts/utils_pypi_supported_tags.py +++ b/etc/scripts/utils_pypi_supported_tags.py @@ -82,11 +82,7 @@ def is_supported_platform_tag(platform_tag): if platform_tag in _allowed_platforms: return True m = _macosx_platform_re.match(platform_tag) - if ( - m - and m.group("major") in _macosx_major_versions - and m.group("arch") in _macosx_arches - ): + if m and m.group("major") in _macosx_major_versions and m.group("arch") in _macosx_arches: return True m = _manylinux_platform_re.match(platform_tag) if m and m.group("arch") in _manylinux_arches: diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index ddbed61..9545db5 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -16,7 +16,7 @@ """ -def load_requirements(requirements_file='requirements.txt', force_pinned=True): +def load_requirements(requirements_file="requirements.txt", force_pinned=True): """ Yield package (name, version) tuples for each requirement in a `requirement` file. Every requirement versions must be pinned if `force_pinned` is True. @@ -36,14 +36,14 @@ def get_required_name_versions(requirement_lines, force_pinned=True): """ for req_line in requirement_lines: req_line = req_line.strip() - if not req_line or req_line.startswith('#'): + if not req_line or req_line.startswith("#"): continue - if '==' not in req_line and force_pinned: - raise Exception(f'Requirement version is not pinned: {req_line}') + if "==" not in req_line and force_pinned: + raise Exception(f"Requirement version is not pinned: {req_line}") name = req_line version = None else: - name, _, version = req_line.partition('==') + name, _, version = req_line.partition("==") name = name.lower().strip() version = version.lower().strip() yield name, version @@ -58,22 +58,22 @@ def parse_requires(requires): if not requires: return [] - requires = [''.join(r.split()) for r in requires if r and r.strip()] + requires = ["".join(r.split()) for r in requires if r and r.strip()] return sorted(requires) -def lock_requirements(requirements_file='requirements.txt', site_packages_dir=None): +def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): """ Freeze and lock current installed requirements and save this to the `requirements_file` requirements file. """ - with open(requirements_file, 'w') as fo: + with open(requirements_file, "w") as fo: fo.write(get_installed_reqs(site_packages_dir=site_packages_dir)) def lock_dev_requirements( - dev_requirements_file='requirements-dev.txt', - main_requirements_file='requirements.txt', + dev_requirements_file="requirements-dev.txt", + main_requirements_file="requirements.txt", site_packages_dir=None, ): """ @@ -89,8 +89,8 @@ def lock_dev_requirements( all_req_nvs = get_required_name_versions(all_req_lines) dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} - new_reqs = '\n'.join(f'{n}=={v}' for n, v in sorted(dev_only_req_nvs.items())) - with open(dev_requirements_file, 'w') as fo: + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + with open(dev_requirements_file, "w") as fo: fo.write(new_reqs) @@ -99,5 +99,5 @@ def get_installed_reqs(site_packages_dir): Return the installed pip requirements as text found in `site_packages_dir` as a text. """ # Also include these packages in the output with --all: wheel, distribute, setuptools, pip - args = ['pip', 'freeze', '--exclude-editable', '--all', '--path', site_packages_dir] - return subprocess.check_output(args, encoding='utf-8') + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] + return subprocess.check_output(args, encoding="utf-8") diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 444b20d..e2778fe 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -87,56 +87,73 @@ TRACE = False # Supported environments -PYTHON_VERSIONS = '36', '37', '38', '39', '310' +PYTHON_VERSIONS = "36", "37", "38", "39", "310" ABIS_BY_PYTHON_VERSION = { - '36':['cp36', 'cp36m'], - '37':['cp37', 'cp37m'], - '38':['cp38', 'cp38m'], - '39':['cp39', 'cp39m'], - '310':['cp310', 'cp310m'], + "36": ["cp36", "cp36m"], + "37": ["cp37", "cp37m"], + "38": ["cp38", "cp38m"], + "39": ["cp39", "cp39m"], + "310": ["cp310", "cp310m"], } PLATFORMS_BY_OS = { - 'linux': [ - 'linux_x86_64', - 'manylinux1_x86_64', - 'manylinux2014_x86_64', - 'manylinux2010_x86_64', - 'manylinux_2_12_x86_64', + "linux": [ + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2014_x86_64", + "manylinux2010_x86_64", + "manylinux_2_12_x86_64", ], - 'macos': [ - 'macosx_10_6_intel', 'macosx_10_6_x86_64', - 'macosx_10_9_intel', 'macosx_10_9_x86_64', - 'macosx_10_10_intel', 'macosx_10_10_x86_64', - 'macosx_10_11_intel', 'macosx_10_11_x86_64', - 'macosx_10_12_intel', 'macosx_10_12_x86_64', - 'macosx_10_13_intel', 'macosx_10_13_x86_64', - 'macosx_10_14_intel', 'macosx_10_14_x86_64', - 'macosx_10_15_intel', 'macosx_10_15_x86_64', - 'macosx_10_15_x86_64', - 'macosx_11_0_x86_64', + "macos": [ + "macosx_10_6_intel", + "macosx_10_6_x86_64", + "macosx_10_9_intel", + "macosx_10_9_x86_64", + "macosx_10_10_intel", + "macosx_10_10_x86_64", + "macosx_10_11_intel", + "macosx_10_11_x86_64", + "macosx_10_12_intel", + "macosx_10_12_x86_64", + "macosx_10_13_intel", + "macosx_10_13_x86_64", + "macosx_10_14_intel", + "macosx_10_14_x86_64", + "macosx_10_15_intel", + "macosx_10_15_x86_64", + "macosx_10_15_x86_64", + "macosx_11_0_x86_64", # 'macosx_11_0_arm64', ], - 'windows': [ - 'win_amd64', + "windows": [ + "win_amd64", ], } -THIRDPARTY_DIR = 'thirdparty' -CACHE_THIRDPARTY_DIR = '.cache/thirdparty' - -REMOTE_LINKS_URL = 'https://thirdparty.aboutcode.org/pypi' - -EXTENSIONS_APP = '.pyz', -EXTENSIONS_SDIST = '.tar.gz', '.tar.bz2', '.zip', '.tar.xz', -EXTENSIONS_INSTALLABLE = EXTENSIONS_SDIST + ('.whl',) -EXTENSIONS_ABOUT = '.ABOUT', '.LICENSE', '.NOTICE', +THIRDPARTY_DIR = "thirdparty" +CACHE_THIRDPARTY_DIR = ".cache/thirdparty" + +REMOTE_LINKS_URL = "https://thirdparty.aboutcode.org/pypi" + +EXTENSIONS_APP = (".pyz",) +EXTENSIONS_SDIST = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) +EXTENSIONS_INSTALLABLE = EXTENSIONS_SDIST + (".whl",) +EXTENSIONS_ABOUT = ( + ".ABOUT", + ".LICENSE", + ".NOTICE", +) EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP -PYPI_SIMPLE_URL = 'https://pypi.org/simple' +PYPI_SIMPLE_URL = "https://pypi.org/simple" -LICENSEDB_API_URL = 'https://scancode-licensedb.aboutcode.org' +LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org" LICENSING = license_expression.Licensing() @@ -149,7 +166,7 @@ def fetch_wheels( environment=None, - requirements_file='requirements.txt', + requirements_file="requirements.txt", allow_unpinned=False, dest_dir=THIRDPARTY_DIR, remote_links_url=REMOTE_LINKS_URL, @@ -179,11 +196,13 @@ def fetch_wheels( force_pinned = False try: - rrp = list(get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - )) + rrp = list( + get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + ) + ) except Exception as e: raise Exception( dict( @@ -196,9 +215,14 @@ def fetch_wheels( fetched_filenames = set() for name, version, package in rrp: if not package: - missed.append((name, version,)) - nv = f'{name}=={version}' if version else name - yield None, f'fetch_wheels: Missing package in remote repo: {nv}' + missed.append( + ( + name, + version, + ) + ) + nv = f"{name}=={version}" if version else name + yield None, f"fetch_wheels: Missing package in remote repo: {nv}" else: fetched_filename = package.fetch_wheel( @@ -214,23 +238,23 @@ def fetch_wheels( if fetched_filename in fetched_filenames: error = None else: - error = f'Failed to fetch' + error = f"Failed to fetch" yield package, error if missed: rr = get_remote_repo() print() - print(f'===> fetch_wheels: Missed some packages') + print(f"===> fetch_wheels: Missed some packages") for n, v in missed: - nv = f'{n}=={v}' if v else n - print(f'Missed package {nv} in remote repo, has only:') + nv = f"{n}=={v}" if v else n + print(f"Missed package {nv} in remote repo, has only:") for pv in rr.get_versions(n): - print(' ', pv) - raise Exception('Missed some packages in remote repo') + print(" ", pv) + raise Exception("Missed some packages in remote repo") def fetch_sources( - requirements_file='requirements.txt', + requirements_file="requirements.txt", allow_unpinned=False, dest_dir=THIRDPARTY_DIR, remote_links_url=REMOTE_LINKS_URL, @@ -258,27 +282,35 @@ def fetch_sources( else: force_pinned = False - rrp = list(get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - )) + rrp = list( + get_required_remote_packages( + requirements_file=requirements_file, + force_pinned=force_pinned, + remote_links_url=remote_links_url, + ) + ) for name, version, package in rrp: if not package: - missed.append((name, name,)) - nv = f'{name}=={version}' if version else name - yield None, f'fetch_sources: Missing package in remote repo: {nv}' + missed.append( + ( + name, + name, + ) + ) + nv = f"{name}=={version}" if version else name + yield None, f"fetch_sources: Missing package in remote repo: {nv}" elif not package.sdist: - yield package, f'Missing sdist in links' + yield package, f"Missing sdist in links" else: fetched = package.fetch_sdist(dest_dir=dest_dir) - error = f'Failed to fetch' if not fetched else None + error = f"Failed to fetch" if not fetched else None yield package, error if missed: - raise Exception(f'Missing source packages in {remote_links_url}', missed) + raise Exception(f"Missing source packages in {remote_links_url}", missed) + ################################################################################ # @@ -291,12 +323,12 @@ def fetch_sources( class NameVer: name = attr.ib( type=str, - metadata=dict(help='Python package name, lowercase and normalized.'), + metadata=dict(help="Python package name, lowercase and normalized."), ) version = attr.ib( type=str, - metadata=dict(help='Python package version string.'), + metadata=dict(help="Python package version string."), ) @property @@ -320,7 +352,7 @@ def standardize_name(name): @property def name_ver(self): - return f'{self.name}-{self.version}' + return f"{self.name}-{self.version}" def sortable_name_version(self): """ @@ -339,146 +371,146 @@ class Distribution(NameVer): # field names that can be updated from another dist of mapping updatable_fields = [ - 'license_expression', - 'copyright', - 'description', - 'homepage_url', - 'primary_language', - 'notice_text', - 'extra_data', + "license_expression", + "copyright", + "description", + "homepage_url", + "primary_language", + "notice_text", + "extra_data", ] filename = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='File name.'), + default="", + metadata=dict(help="File name."), ) path_or_url = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Path or download URL.'), + default="", + metadata=dict(help="Path or download URL."), ) sha256 = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='SHA256 checksum.'), + default="", + metadata=dict(help="SHA256 checksum."), ) sha1 = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='SHA1 checksum.'), + default="", + metadata=dict(help="SHA1 checksum."), ) md5 = attr.ib( repr=False, type=int, default=0, - metadata=dict(help='MD5 checksum.'), + metadata=dict(help="MD5 checksum."), ) type = attr.ib( repr=False, type=str, - default='pypi', - metadata=dict(help='Package type'), + default="pypi", + metadata=dict(help="Package type"), ) namespace = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Package URL namespace'), + default="", + metadata=dict(help="Package URL namespace"), ) qualifiers = attr.ib( repr=False, type=dict, default=attr.Factory(dict), - metadata=dict(help='Package URL qualifiers'), + metadata=dict(help="Package URL qualifiers"), ) subpath = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Package URL subpath'), + default="", + metadata=dict(help="Package URL subpath"), ) size = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Size in bytes.'), + default="", + metadata=dict(help="Size in bytes."), ) primary_language = attr.ib( repr=False, type=str, - default='Python', - metadata=dict(help='Primary Programming language.'), + default="Python", + metadata=dict(help="Primary Programming language."), ) description = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Description.'), + default="", + metadata=dict(help="Description."), ) homepage_url = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Homepage URL'), + default="", + metadata=dict(help="Homepage URL"), ) notes = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Notes.'), + default="", + metadata=dict(help="Notes."), ) copyright = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Copyright.'), + default="", + metadata=dict(help="Copyright."), ) license_expression = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='License expression'), + default="", + metadata=dict(help="License expression"), ) licenses = attr.ib( repr=False, type=list, default=attr.Factory(list), - metadata=dict(help='List of license mappings.'), + metadata=dict(help="List of license mappings."), ) notice_text = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Notice text'), + default="", + metadata=dict(help="Notice text"), ) extra_data = attr.ib( repr=False, type=dict, default=attr.Factory(dict), - metadata=dict(help='Extra data'), + metadata=dict(help="Extra data"), ) @property @@ -490,14 +522,14 @@ def package_url(self): @property def download_url(self): - if self.path_or_url and self.path_or_url.startswith('https://'): + if self.path_or_url and self.path_or_url.startswith("https://"): return self.path_or_url else: return self.get_best_download_url() @property def about_filename(self): - return f'{self.filename}.ABOUT' + return f"{self.filename}.ABOUT" def has_about_file(self, dest_dir=THIRDPARTY_DIR): return os.path.exists(os.path.join(dest_dir, self.about_filename)) @@ -508,7 +540,7 @@ def about_download_url(self): @property def notice_filename(self): - return f'{self.filename}.NOTICE' + return f"{self.filename}.NOTICE" @property def notice_download_url(self): @@ -521,16 +553,21 @@ def from_path_or_url(cls, path_or_url): `path_or_url` string. Raise an exception if this is not a valid filename. """ - filename = os.path.basename(path_or_url.strip('/')) + filename = os.path.basename(path_or_url.strip("/")) dist = cls.from_filename(filename) dist.path_or_url = path_or_url return dist @classmethod def get_dist_class(cls, filename): - if filename.endswith('.whl'): + if filename.endswith(".whl"): return Wheel - elif filename.endswith(('.zip', '.tar.gz',)): + elif filename.endswith( + ( + ".zip", + ".tar.gz", + ) + ): return Sdist raise InvalidDistributionFilename(filename) @@ -548,7 +585,7 @@ def from_data(cls, data, keep_extra=False): """ Return a distribution built from a `data` mapping. """ - filename = data['filename'] + filename = data["filename"] dist = cls.from_filename(filename) dist.update(data, keep_extra=keep_extra) return dist @@ -560,16 +597,20 @@ def from_dist(cls, data, dist): from another dist Distribution. Return None if it cannot be created """ # We can only create from a dist of the same package - has_same_key_fields = all(data.get(kf) == getattr(dist, kf, None) - for kf in ('type', 'namespace', 'name') + has_same_key_fields = all( + data.get(kf) == getattr(dist, kf, None) for kf in ("type", "namespace", "name") ) if not has_same_key_fields: - print(f'Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}') + print( + f"Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}" + ) return - has_key_field_values = all(data.get(kf) for kf in ('type', 'name', 'version')) + has_key_field_values = all(data.get(kf) for kf in ("type", "name", "version")) if not has_key_field_values: - print(f'Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}') + print( + f"Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}" + ) return data = dict(data) @@ -583,7 +624,7 @@ def build_remote_download_url(cls, filename, base_url=REMOTE_LINKS_URL): """ Return a direct download URL for a file in our remote repo """ - return f'{base_url}/{filename}' + return f"{base_url}/{filename}" def get_best_download_url(self): """ @@ -656,7 +697,7 @@ def has_key_metadata(self): """ Return True if this distribution has key metadata required for basic attribution. """ - if self.license_expression == 'public-domain': + if self.license_expression == "public-domain": # copyright not needed return True return self.license_expression and self.copyright and self.path_or_url @@ -677,7 +718,7 @@ def to_about(self): name=self.name, namespace=self.namespace, notes=self.notes, - notice_file=self.notice_filename if self.notice_text else '', + notice_file=self.notice_filename if self.notice_text else "", package_url=self.package_url, primary_language=self.primary_language, qualifiers=self.qualifiers, @@ -695,7 +736,7 @@ def to_dict(self): """ Return a mapping data from this distribution. """ - return {k: v for k, v in attr.asdict(self).items() if v} + return {k: v for k, v in attr.asdict(self).items() if v} def save_about_and_notice_files(self, dest_dir=THIRDPARTY_DIR): """ @@ -710,8 +751,9 @@ def save_if_modified(location, content): if existing_content == content: return False - if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}') - with open(location, 'w') as fo: + if TRACE: + print(f"Saving ABOUT (and NOTICE) files for: {self}") + with open(location, "w") as fo: fo.write(content) return True @@ -750,26 +792,26 @@ def load_about_data(self, about_filename_or_data=None, dest_dir=THIRDPARTY_DIR): else: about_data = about_filename_or_data - md5 = about_data.pop('checksum_md5', None) + md5 = about_data.pop("checksum_md5", None) if md5: - about_data['md5'] = md5 - sha1 = about_data.pop('checksum_sha1', None) + about_data["md5"] = md5 + sha1 = about_data.pop("checksum_sha1", None) if sha1: - about_data['sha1'] = sha1 - sha256 = about_data.pop('checksum_sha256', None) + about_data["sha1"] = sha1 + sha256 = about_data.pop("checksum_sha256", None) if sha256: - about_data['sha256'] = sha256 + about_data["sha256"] = sha256 - about_data.pop('about_resource', None) - notice_text = about_data.pop('notice_text', None) - notice_file = about_data.pop('notice_file', None) + about_data.pop("about_resource", None) + notice_text = about_data.pop("notice_text", None) + notice_file = about_data.pop("notice_file", None) if notice_text: - about_data['notice_text'] = notice_text + about_data["notice_text"] = notice_text elif notice_file: notice_loc = os.path.join(dest_dir, notice_file) if os.path.exists(notice_loc): with open(notice_loc) as fi: - about_data['notice_text'] = fi.read() + about_data["notice_text"] = fi.read() return self.update(about_data, keep_extra=True) def load_remote_about_data(self): @@ -786,14 +828,14 @@ def load_remote_about_data(self): return False about_data = saneyaml.load(about_text) - notice_file = about_data.pop('notice_file', None) + notice_file = about_data.pop("notice_file", None) if notice_file: try: notice_text = fetch_content_from_path_or_url_through_cache(self.notice_download_url) if notice_text: - about_data['notice_text'] = notice_text + about_data["notice_text"] = notice_text except RemoteNotFetchedException: - print(f'Failed to fetch NOTICE file: {self.notice_download_url}') + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") return self.load_about_data(about_data) def get_checksums(self, dest_dir=THIRDPARTY_DIR): @@ -803,7 +845,7 @@ def get_checksums(self, dest_dir=THIRDPARTY_DIR): """ dist_loc = os.path.join(dest_dir, self.filename) if os.path.exists(dist_loc): - return multi_checksums(dist_loc, checksum_names=('md5', 'sha1', 'sha256')) + return multi_checksums(dist_loc, checksum_names=("md5", "sha1", "sha256")) else: return {} @@ -819,7 +861,7 @@ def validate_checksums(self, dest_dir=THIRDPARTY_DIR): checksums computed for this dist filename is `dest_dir`. """ real_checksums = self.get_checksums(dest_dir) - for csk in ('md5', 'sha1', 'sha256'): + for csk in ("md5", "sha1", "sha256"): csv = getattr(self, csk) rcv = real_checksums.get(csk) if csv and rcv and csv != rcv: @@ -830,14 +872,14 @@ def get_pip_hash(self): """ Return a pip hash option string as used in requirements for this dist. """ - assert self.sha256, f'Missinh SHA256 for dist {self}' - return f'--hash=sha256:{self.sha256}' + assert self.sha256, f"Missinh SHA256 for dist {self}" + return f"--hash=sha256:{self.sha256}" def get_license_keys(self): try: keys = LICENSING.license_keys(self.license_expression, unique=True, simple=True) except license_expression.ExpressionParseError: - return ['unknown'] + return ["unknown"] return keys def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): @@ -847,19 +889,18 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): """ paths_or_urls = get_remote_repo().links errors = [] - extra_lic_names = [l.get('file') for l in self.extra_data.get('licenses', {})] - extra_lic_names += [self.extra_data.get('license_file')] - extra_lic_names = [ln for ln in extra_lic_names if ln] - lic_names = [ f'{key}.LICENSE' for key in self.get_license_keys()] - for filename in lic_names + extra_lic_names: + extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] + extra_lic_names += [self.extra_data.get("license_file")] + extra_lic_names = [ln for ln in extra_lic_names if ln] + lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] + for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) if os.path.exists(floc): continue try: # try remotely first - lic_url = get_link_for_filename( - filename=filename, paths_or_urls=paths_or_urls) + lic_url = get_link_for_filename(filename=filename, paths_or_urls=paths_or_urls) fetch_and_save_path_or_url( filename=filename, @@ -867,19 +908,21 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): path_or_url=lic_url, as_text=True, ) - if TRACE: print(f'Fetched license from remote: {lic_url}') + if TRACE: + print(f"Fetched license from remote: {lic_url}") except: try: # try licensedb second - lic_url = f'{LICENSEDB_API_URL}/{filename}' + lic_url = f"{LICENSEDB_API_URL}/{filename}" fetch_and_save_path_or_url( filename=filename, dest_dir=dest_dir, path_or_url=lic_url, as_text=True, ) - if TRACE: print(f'Fetched license from licensedb: {lic_url}') + if TRACE: + print(f"Fetched license from licensedb: {lic_url}") except: msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' @@ -893,14 +936,19 @@ def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): Return the text of the first PKG-INFO or METADATA file found in the archive of this Distribution in `dest_dir`. Return None if not found. """ - fmt = 'zip' if self.filename.endswith('.whl') else None + fmt = "zip" if self.filename.endswith(".whl") else None dist = os.path.join(dest_dir, self.filename) - with tempfile.TemporaryDirectory(prefix='pypi-tmp-extract') as td: + with tempfile.TemporaryDirectory(prefix="pypi-tmp-extract") as td: shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) # NOTE: we only care about the first one found in the dist # which may not be 100% right for pi in fileutils.resource_iter(location=td, with_dirs=False): - if pi.endswith(('PKG-INFO', 'METADATA',)): + if pi.endswith( + ( + "PKG-INFO", + "METADATA", + ) + ): with open(pi) as fi: return fi.read() @@ -911,31 +959,33 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): """ pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) if not pkginfo_text: - print(f'!!!!PKG-INFO not found in {self.filename}') + print(f"!!!!PKG-INFO not found in {self.filename}") return raw_data = email.message_from_string(pkginfo_text) - classifiers = raw_data.get_all('Classifier') or [] + classifiers = raw_data.get_all("Classifier") or [] - declared_license = [raw_data['License']] + [c for c in classifiers if c.startswith('License')] + declared_license = [raw_data["License"]] + [ + c for c in classifiers if c.startswith("License") + ] license_expression = compute_normalized_license_expression(declared_license) - other_classifiers = [c for c in classifiers if not c.startswith('License')] + other_classifiers = [c for c in classifiers if not c.startswith("License")] - holder = raw_data['Author'] - holder_contact = raw_data['Author-email'] - copyright_statement = f'Copyright (c) {holder} <{holder_contact}>' + holder = raw_data["Author"] + holder_contact = raw_data["Author-email"] + copyright_statement = f"Copyright (c) {holder} <{holder_contact}>" pkginfo_data = dict( - name=raw_data['Name'], + name=raw_data["Name"], declared_license=declared_license, - version=raw_data['Version'], - description=raw_data['Summary'], - homepage_url=raw_data['Home-page'], + version=raw_data["Version"], + description=raw_data["Summary"], + homepage_url=raw_data["Home-page"], copyright=copyright_statement, license_expression=license_expression, holder=holder, holder_contact=holder_contact, - keywords=raw_data['Keywords'], + keywords=raw_data["Keywords"], classifiers=other_classifiers, ) @@ -949,10 +999,7 @@ def update_from_other_dist(self, dist): def get_updatable_data(self, data=None): data = data or self.to_dict() - return { - k: v for k, v in data.items() - if v and k in self.updatable_fields - } + return {k: v for k, v in data.items() if v and k in self.updatable_fields} def update(self, data, overwrite=False, keep_extra=True): """ @@ -961,20 +1008,21 @@ def update(self, data, overwrite=False, keep_extra=True): Return True if any data was updated, False otherwise. Raise an exception if there are key data conflicts. """ - package_url = data.get('package_url') + package_url = data.get("package_url") if package_url: purl_from_data = packageurl.PackageURL.from_string(package_url) purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: print( - f'Invalid dist update attempt, no same same purl with dist: ' - f'{self} using data {data}.') + f"Invalid dist update attempt, no same same purl with dist: " + f"{self} using data {data}." + ) return - data.pop('about_resource', None) - dl = data.pop('download_url', None) + data.pop("about_resource", None) + dl = data.pop("download_url", None) if dl: - data['path_or_url'] = dl + data["path_or_url"] = dl updated = False extra = {} @@ -990,7 +1038,7 @@ def update(self, data, overwrite=False, keep_extra=True): try: setattr(self, k, v) except Exception as e: - raise Exception(f'{self}, {k}, {v}') from e + raise Exception(f"{self}, {k}, {v}") from e updated = True elif keep_extra: @@ -1013,8 +1061,8 @@ class Sdist(Distribution): extension = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='File extension, including leading dot.'), + default="", + metadata=dict(help="File extension, including leading dot."), ) @classmethod @@ -1034,13 +1082,13 @@ def from_filename(cls, filename): if not extension or not name_ver: raise InvalidDistributionFilename(filename) - name, _, version = name_ver.rpartition('-') + name, _, version = name_ver.rpartition("-") if not name or not version: raise InvalidDistributionFilename(filename) return cls( - type='pypi', + type="pypi", name=name, version=version, extension=extension, @@ -1052,7 +1100,7 @@ def to_filename(self): Return an sdist filename reconstructed from its fields (that may not be the same as the original filename.) """ - return f'{self.name}-{self.version}.{self.extension}' + return f"{self.name}-{self.version}.{self.extension}" @attr.attributes @@ -1097,38 +1145,38 @@ class Wheel(Distribution): r"""^(?P(?P.+?)-(?P.*?)) ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) \.whl)$""", - re.VERBOSE + re.VERBOSE, ).match build = attr.ib( type=str, - default='', - metadata=dict(help='Python wheel build.'), + default="", + metadata=dict(help="Python wheel build."), ) python_versions = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of wheel Python version tags.'), + metadata=dict(help="List of wheel Python version tags."), ) abis = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of wheel ABI tags.'), + metadata=dict(help="List of wheel ABI tags."), ) platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of wheel platform tags.'), + metadata=dict(help="List of wheel platform tags."), ) tags = attr.ib( repr=False, type=set, default=attr.Factory(set), - metadata=dict(help='Set of all tags for this wheel.'), + metadata=dict(help="Set of all tags for this wheel."), ) @classmethod @@ -1141,24 +1189,23 @@ def from_filename(cls, filename): if not wheel_info: raise InvalidDistributionFilename(filename) - name = wheel_info.group('name').replace('_', '-') + name = wheel_info.group("name").replace("_", "-") # we'll assume "_" means "-" due to wheel naming scheme # (https://github.com/pypa/pip/issues/1150) - version = wheel_info.group('ver').replace('_', '-') - build = wheel_info.group('build') - python_versions = wheel_info.group('pyvers').split('.') - abis = wheel_info.group('abis').split('.') - platforms = wheel_info.group('plats').split('.') + version = wheel_info.group("ver").replace("_", "-") + build = wheel_info.group("build") + python_versions = wheel_info.group("pyvers").split(".") + abis = wheel_info.group("abis").split(".") + platforms = wheel_info.group("plats").split(".") # All the tag combinations from this file tags = { - packaging_tags.Tag(x, y, z) for x in python_versions - for y in abis for z in platforms + packaging_tags.Tag(x, y, z) for x in python_versions for y in abis for z in platforms } return cls( filename=filename, - type='pypi', + type="pypi", name=name, version=version, build=build, @@ -1179,18 +1226,18 @@ def is_supported_by_environment(self, environment): Return True if this wheel is compatible with the Environment `environment`. """ - return not self.is_supported_by_tags(environment.tags) + return not self.is_supported_by_tags(environment.tags) def to_filename(self): """ Return a wheel filename reconstructed from its fields (that may not be the same as the original filename.) """ - build = f'-{self.build}' if self.build else '' - pyvers = '.'.join(self.python_versions) - abis = '.'.join(self.abis) - plats = '.'.join(self.platforms) - return f'{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl' + build = f"-{self.build}" if self.build else "" + pyvers = ".".join(self.python_versions) + abis = ".".join(self.abis) + plats = ".".join(self.platforms) + return f"{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl" def is_pure(self): """ @@ -1216,11 +1263,7 @@ def is_pure(self): >>> Wheel.from_filename('future-0.16.0-py3-cp36m-any.whl').is_pure() False """ - return ( - 'py3' in self.python_versions - and 'none' in self.abis - and 'any' in self.platforms - ) + return "py3" in self.python_versions and "none" in self.abis and "any" in self.platforms def is_pure_wheel(filename): @@ -1236,18 +1279,19 @@ class PypiPackage(NameVer): A Python package with its "distributions", e.g. wheels and source distribution , ABOUT files and licenses or notices. """ + sdist = attr.ib( repr=False, type=str, - default='', - metadata=dict(help='Sdist source distribution for this package.'), + default="", + metadata=dict(help="Sdist source distribution for this package."), ) wheels = attr.ib( repr=False, type=list, default=attr.Factory(list), - metadata=dict(help='List of Wheel for this package'), + metadata=dict(help="List of Wheel for this package"), ) @property @@ -1256,7 +1300,7 @@ def specifier(self): A requirement specifier for this package """ if self.version: - return f'{self.name}=={self.version}' + return f"{self.name}=={self.version}" else: return self.name @@ -1268,7 +1312,7 @@ def specifier_with_hashes(self): """ items = [self.specifier] items += [d.get_pip_hashes() for d in self.get_distributions()] - return ' \\\n '.join(items) + return " \\\n ".join(items) def get_supported_wheels(self, environment): """ @@ -1314,7 +1358,7 @@ def package_from_dists(cls, dists): if dist.normalized_name != normalized_name or dist.version != version: if TRACE: print( - f' Skipping inconsistent dist name and version: {dist} ' + f" Skipping inconsistent dist name and version: {dist} " f'Expected instead package name: {normalized_name} and version: "{version}"' ) continue @@ -1326,7 +1370,7 @@ def package_from_dists(cls, dists): package.wheels.append(dist) else: - raise Exception(f'Unknown distribution type: {dist}') + raise Exception(f"Unknown distribution type: {dist}") return package @@ -1348,7 +1392,8 @@ def packages_from_many_paths_or_urls(cls, paths_or_urls): dists = NameVer.sorted(dists) for _projver, dists_of_package in itertools.groupby( - dists, key=NameVer.sortable_name_version, + dists, + key=NameVer.sortable_name_version, ): yield PypiPackage.package_from_dists(dists_of_package) @@ -1409,7 +1454,7 @@ def get_name_version(cls, name, version, packages): if len(nvs) == 1: return nvs[0] - raise Exception(f'More than one PypiPackage with {name}=={version}') + raise Exception(f"More than one PypiPackage with {name}=={version}") def fetch_wheel( self, @@ -1457,17 +1502,19 @@ def fetch_sdist(self, dest_dir=THIRDPARTY_DIR): """ if self.sdist: assert self.sdist.filename - if TRACE: print('Fetching source for package:', self.name, self.version) + if TRACE: + print("Fetching source for package:", self.name, self.version) fetch_and_save_path_or_url( filename=self.sdist.filename, dest_dir=dest_dir, path_or_url=self.sdist.path_or_url, as_text=False, ) - if TRACE: print(' --> file:', self.sdist.filename) + if TRACE: + print(" --> file:", self.sdist.filename) return self.sdist.filename else: - print(f'Missing sdist for: {self.name}=={self.version}') + print(f"Missing sdist for: {self.name}=={self.version}") return False def delete_files(self, dest_dir=THIRDPARTY_DIR): @@ -1481,10 +1528,10 @@ def delete_files(self, dest_dir=THIRDPARTY_DIR): if not to_delete: continue tdfn = to_delete.filename - for deletable in [tdfn, f'{tdfn}.ABOUT', f'{tdfn}.NOTICE']: + for deletable in [tdfn, f"{tdfn}.ABOUT", f"{tdfn}.NOTICE"]: target = os.path.join(dest_dir, deletable) if os.path.exists(target): - print(f'Deleting outdated {target}') + print(f"Deleting outdated {target}") fileutils.delete(target) @classmethod @@ -1528,7 +1575,7 @@ def get_dists(cls, paths_or_urls): yield Distribution.from_path_or_url(path_or_url) except InvalidDistributionFilename: if TRACE: - print(f'Skipping invalid distribution from: {path_or_url}') + print(f"Skipping invalid distribution from: {path_or_url}") continue def get_distributions(self): @@ -1562,42 +1609,42 @@ class Environment: python_version = attr.ib( type=str, - default='', - metadata=dict(help='Python version supported by this environment.'), + default="", + metadata=dict(help="Python version supported by this environment."), ) operating_system = attr.ib( type=str, - default='', - metadata=dict(help='operating system supported by this environment.'), + default="", + metadata=dict(help="operating system supported by this environment."), ) implementation = attr.ib( type=str, - default='cp', - metadata=dict(help='Python implementation supported by this environment.'), + default="cp", + metadata=dict(help="Python implementation supported by this environment."), ) abis = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of ABI tags supported by this environment.'), + metadata=dict(help="List of ABI tags supported by this environment."), ) platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of platform tags supported by this environment.'), + metadata=dict(help="List of platform tags supported by this environment."), ) @classmethod def from_pyver_and_os(cls, python_version, operating_system): - if '.' in python_version: - python_version = ''.join(python_version.split('.')) + if "." in python_version: + python_version = "".join(python_version.split(".")) return cls( python_version=python_version, - implementation='cp', + implementation="cp", abis=ABIS_BY_PYTHON_VERSION[python_version], platforms=PLATFORMS_BY_OS[operating_system], operating_system=operating_system, @@ -1608,24 +1655,30 @@ def get_pip_cli_options(self): Return a list of pip command line options for this environment. """ options = [ - '--python-version', self.python_version, - '--implementation', self.implementation, - '--abi', self.abi, + "--python-version", + self.python_version, + "--implementation", + self.implementation, + "--abi", + self.abi, ] for platform in self.platforms: - options.extend(['--platform', platform]) + options.extend(["--platform", platform]) return options def tags(self): """ Return a set of all the PEP425 tags supported by this environment. """ - return set(utils_pip_compatibility_tags.get_supported( - version=self.python_version or None, - impl=self.implementation or None, - platforms=self.platforms or None, - abis=self.abis or None, - )) + return set( + utils_pip_compatibility_tags.get_supported( + version=self.python_version or None, + impl=self.implementation or None, + platforms=self.platforms or None, + abis=self.abis or None, + ) + ) + ################################################################################ # @@ -1643,15 +1696,13 @@ class Repository: packages_by_normalized_name = attr.ib( type=dict, default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help= - 'Mapping of {package name: [package objects]} available in this repo'), + metadata=dict(help="Mapping of {package name: [package objects]} available in this repo"), ) packages_by_normalized_name_version = attr.ib( type=dict, default=attr.Factory(dict), - metadata=dict(help= - 'Mapping of {(name, version): package object} available in this repo'), + metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), ) def get_links(self, *args, **kwargs): @@ -1684,16 +1735,17 @@ class LinksRepository(Repository): Python wheels and sdist or a remote URL to an HTML with links to these. (e.g. suitable for use with pip --find-links). """ + path_or_url = attr.ib( type=str, - default='', - metadata=dict(help='Package directory path or URL'), + default="", + metadata=dict(help="Package directory path or URL"), ) links = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help='List of links available in this repo'), + metadata=dict(help="List of links available in this repo"), ) def __attrs_post_init__(self): @@ -1725,16 +1777,17 @@ class PypiRepository(Repository): Represents the public PyPI simple index. It is populated lazily based on requested packages names """ + simple_url = attr.ib( type=str, default=PYPI_SIMPLE_URL, - metadata=dict(help='Base PyPI simple URL for this index.'), + metadata=dict(help="Base PyPI simple URL for this index."), ) links_by_normalized_name = attr.ib( type=dict, default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help='Mapping of {package name: [links]} available in this repo'), + metadata=dict(help="Mapping of {package name: [links]} available in this repo"), ) def _fetch_links(self, name): @@ -1759,7 +1812,7 @@ def _populate_links_and_packages(self, name): def get_links(self, name, *args, **kwargs): name = name and NameVer.normalize_name(name) self._populate_links_and_packages(name) - return self.links_by_normalized_name.get(name, []) + return self.links_by_normalized_name.get(name, []) def get_versions(self, name): name = name and NameVer.normalize_name(name) @@ -1772,6 +1825,7 @@ def get_latest_version(self, name): def get_package(self, name, version): return PypiPackage.get_name_version(name, version, self.get_versions(name)) + ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1807,7 +1861,7 @@ def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL): try: return get_remote_repo(remote_links_url).get_package(name, version) except RemoteNotFetchedException as e: - print(f'Failed to fetch remote package info: {e}') + print(f"Failed to fetch remote package info: {e}") _PYPI_REPO = None @@ -1827,7 +1881,8 @@ def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL): try: return get_pypi_repo(pypi_simple_url).get_package(name, version) except RemoteNotFetchedException as e: - print(f'Failed to fetch remote package info: {e}') + print(f"Failed to fetch remote package info: {e}") + ################################################################################ # @@ -1856,12 +1911,12 @@ def get(self, path_or_url, as_text=True): Get a file from a `path_or_url` through the cache. `path_or_url` can be a path or a URL to a file. """ - filename = os.path.basename(path_or_url.strip('/')) + filename = os.path.basename(path_or_url.strip("/")) cached = os.path.join(self.directory, filename) if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) - wmode = 'w' if as_text else 'wb' + wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) return content @@ -1873,7 +1928,7 @@ def put(self, filename, content): Put in the cache the `content` of `filename`. """ cached = os.path.join(self.directory, filename) - wmode = 'wb' if isinstance(content, bytes) else 'w' + wmode = "wb" if isinstance(content, bytes) else "w" with open(cached, wmode) as fo: fo.write(content) @@ -1883,18 +1938,19 @@ def get_file_content(path_or_url, as_text=True): Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ - if (path_or_url.startswith('file://') - or (path_or_url.startswith('/') and os.path.exists(path_or_url)) + if path_or_url.startswith("file://") or ( + path_or_url.startswith("/") and os.path.exists(path_or_url) ): return get_local_file_content(path=path_or_url, as_text=as_text) - elif path_or_url.startswith('https://'): - if TRACE: print(f'Fetching: {path_or_url}') + elif path_or_url.startswith("https://"): + if TRACE: + print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content else: - raise Exception(f'Unsupported URL scheme: {path_or_url}') + raise Exception(f"Unsupported URL scheme: {path_or_url}") def get_local_file_content(path, as_text=True): @@ -1902,10 +1958,10 @@ def get_local_file_content(path, as_text=True): Return the content at `url` as text. Return the content as bytes is `as_text` is False. """ - if path.startswith('file://'): + if path.startswith("file://"): path = path[7:] - mode = 'r' if as_text else 'rb' + mode = "r" if as_text else "rb" with open(path, mode) as fo: return fo.read() @@ -1914,7 +1970,13 @@ class RemoteNotFetchedException(Exception): pass -def get_remote_file_content(url, as_text=True, headers_only=False, headers=None, _delay=0,): +def get_remote_file_content( + url, + as_text=True, + headers_only=False, + headers=None, + _delay=0, +): """ Fetch and return a tuple of (headers, content) at `url`. Return content as a text string if `as_text` is True. Otherwise return the content as bytes. @@ -1944,7 +2006,7 @@ def get_remote_file_content(url, as_text=True, headers_only=False, headers=None, ) else: - raise RemoteNotFetchedException(f'Failed HTTP request from {url} with {status}') + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") if headers_only: return response.headers, None @@ -1952,7 +2014,11 @@ def get_remote_file_content(url, as_text=True, headers_only=False, headers=None, return response.headers, response.text if as_text else response.content -def get_url_content_if_modified(url, md5, _delay=0,): +def get_url_content_if_modified( + url, + md5, + _delay=0, +): """ Return fetched content bytes at `url` or None if the md5 has not changed. Retries multiple times to fetch if there is a HTTP 429 throttling response @@ -1962,7 +2028,7 @@ def get_url_content_if_modified(url, md5, _delay=0,): headers = None if md5: etag = f'"{md5}"' - headers = {'If-None-Match': f'{etag}'} + headers = {"If-None-Match": f"{etag}"} # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may @@ -1979,7 +2045,7 @@ def get_url_content_if_modified(url, md5, _delay=0,): return None elif status != requests.codes.ok: # NOQA - raise RemoteNotFetchedException(f'Failed HTTP request from {url} with {status}') + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") return response.content @@ -2045,11 +2111,12 @@ def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, th content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text, cache=None) output = os.path.join(dest_dir, filename) - wmode = 'w' if as_text else 'wb' + wmode = "w" if as_text else "wb" with open(output, wmode) as fo: fo.write(content) return content + ################################################################################ # # Sync and fix local thirdparty directory for various issues and gaps @@ -2070,29 +2137,34 @@ def fetch_missing_sources(dest_dir=THIRDPARTY_DIR): for package in local_packages: if not package.sdist: - print(f'Finding sources for: {package.name}=={package.version}: ', end='') + print(f"Finding sources for: {package.name}=={package.version}: ", end="") try: - pypi_package = pypi_repo.get_package( - name=package.name, version=package.version) + pypi_package = pypi_repo.get_package(name=package.name, version=package.version) if pypi_package and pypi_package.sdist: - print(f'Fetching sources from Pypi') + print(f"Fetching sources from Pypi") pypi_package.fetch_sdist(dest_dir=dest_dir) continue else: remote_package = remote_repo.get_package( - name=package.name, version=package.version) + name=package.name, version=package.version + ) if remote_package and remote_package.sdist: - print(f'Fetching sources from Remote') + print(f"Fetching sources from Remote") remote_package.fetch_sdist(dest_dir=dest_dir) continue except RemoteNotFetchedException as e: - print(f'Failed to fetch remote package info: {e}') + print(f"Failed to fetch remote package info: {e}") - print(f'No sources found') - not_found.append((package.name, package.version,)) + print(f"No sources found") + not_found.append( + ( + package.name, + package.version, + ) + ) return not_found @@ -2125,7 +2197,12 @@ def fetch_missing_wheels( if filename: fetched_filenames.add(filename) else: - not_fetched.append((package, envt,)) + not_fetched.append( + ( + package, + envt, + ) + ) return not_fetched @@ -2145,8 +2222,7 @@ def build_missing_wheels( not_built = [] built_filenames = [] - packages_and_envts = itertools.groupby( - sorted(packages_and_envts), key=operator.itemgetter(0)) + packages_and_envts = itertools.groupby(sorted(packages_and_envts), key=operator.itemgetter(0)) for package, pkg_envts in packages_and_envts: @@ -2164,25 +2240,27 @@ def build_missing_wheels( verbose=False, dest_dir=dest_dir, ) - print('.') + print(".") except Exception as e: import traceback - print('#############################################################') - print('############# WHEEL BUILD FAILED ######################') + + print("#############################################################") + print("############# WHEEL BUILD FAILED ######################") traceback.print_exc() print() - print('#############################################################') + print("#############################################################") if not built: for envt in pkg_envts: not_built.append((package, envt)) else: for bfn in built: - print(f' --> Built wheel: {bfn}') + print(f" --> Built wheel: {bfn}") built_filenames.append(bfn) return not_built, built_filenames + ################################################################################ # # Functions to handle remote or local repo used to "find-links" @@ -2191,7 +2269,7 @@ def build_missing_wheels( def get_paths_or_urls(links_url): - if links_url.startswith('https:'): + if links_url.startswith("https:"): paths_or_urls = find_links_from_release_url(links_url) else: paths_or_urls = find_links_from_dir(links_url) @@ -2217,14 +2295,15 @@ def find_links_from_release_url(links_url=REMOTE_LINKS_URL): URL that starts with the `prefix` string and ends with any of the extension in the list of `extensions` strings. Use the `base_url` to prefix the links. """ - if TRACE: print(f'Finding links for {links_url}') + if TRACE: + print(f"Finding links for {links_url}") plinks_url = urllib.parse.urlparse(links_url) - base_url = urllib.parse.SplitResult( - plinks_url.scheme, plinks_url.netloc, '', '', '').geturl() + base_url = urllib.parse.SplitResult(plinks_url.scheme, plinks_url.netloc, "", "", "").geturl() - if TRACE: print(f'Base URL {base_url}') + if TRACE: + print(f"Base URL {base_url}") _headers, text = get_remote_file_content(links_url) links = [] @@ -2238,19 +2317,21 @@ def find_links_from_release_url(links_url=REMOTE_LINKS_URL): # full URL kept as-is url = link - if plink.path.startswith('/'): + if plink.path.startswith("/"): # absolute link - url = f'{base_url}{link}' + url = f"{base_url}{link}" else: # relative link - url = f'{links_url}/{link}' + url = f"{links_url}/{link}" - if TRACE: print(f'Adding URL: {url}') + if TRACE: + print(f"Adding URL: {url}") links.append(url) - if TRACE: print(f'Found {len(links)} links at {links_url}') + if TRACE: + print(f"Found {len(links)} links at {links_url}") return links @@ -2259,19 +2340,20 @@ def find_pypi_links(name, simple_url=PYPI_SIMPLE_URL): Return a list of download link URLs found in a PyPI simple index for package name. with the list of `extensions` strings. Use the `simple_url` PyPI url. """ - if TRACE: print(f'Finding links for {simple_url}') + if TRACE: + print(f"Finding links for {simple_url}") name = name and NameVer.normalize_name(name) - simple_url = simple_url.strip('/') - simple_url = f'{simple_url}/{name}' + simple_url = simple_url.strip("/") + simple_url = f"{simple_url}/{name}" _headers, text = get_remote_file_content(simple_url) links = get_links(text) # TODO: keep sha256 - links = [l.partition('#sha256=') for l in links] + links = [l.partition("#sha256=") for l in links] links = [url for url, _, _sha256 in links] links = [l for l in links if l.endswith(EXTENSIONS)] - return links + return links def get_link_for_filename(filename, paths_or_urls): @@ -2280,13 +2362,14 @@ def get_link_for_filename(filename, paths_or_urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [l for l in paths_or_urls if l.endswith(f'/{filename}')] + path_or_url = [l for l in paths_or_urls if l.endswith(f"/{filename}")] if not path_or_url: - raise Exception(f'Missing link to file: {filename}') + raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: - raise Exception(f'Multiple links to file: {filename}: \n' + '\n'.join(path_or_url)) + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) return path_or_url[0] + ################################################################################ # # Requirements processing @@ -2308,12 +2391,16 @@ def get_required_packages(required_name_versions): """ remote_repo = get_remote_repo() - remote_packages = {(name, version): remote_repo.get_package(name, version) - for name, version in required_name_versions} + remote_packages = { + (name, version): remote_repo.get_package(name, version) + for name, version in required_name_versions + } pypi_repo = get_pypi_repo() - pypi_packages = {(name, version): pypi_repo.get_package(name, version) - for name, version in required_name_versions} + pypi_packages = { + (name, version): pypi_repo.get_package(name, version) + for name, version in required_name_versions + } # remove any empty package (e.g. that do not exist in some place) remote_packages = {nv: p for nv, p in remote_packages.items() if p} @@ -2329,7 +2416,7 @@ def get_required_packages(required_name_versions): def get_required_remote_packages( - requirements_file='requirements.txt', + requirements_file="requirements.txt", force_pinned=True, remote_links_url=REMOTE_LINKS_URL, ): @@ -2344,11 +2431,11 @@ def get_required_remote_packages( force_pinned=force_pinned, ) - if remote_links_url.startswith('https://'): + if remote_links_url.startswith("https://"): repo = get_remote_repo(remote_links_url=remote_links_url) else: # a local path - assert os.path.exists(remote_links_url), f'Path does not exist: {remote_links_url}' + assert os.path.exists(remote_links_url), f"Path does not exist: {remote_links_url}" repo = get_local_repo(directory=remote_links_url) for name, version in required_name_versions: @@ -2358,7 +2445,7 @@ def get_required_remote_packages( yield name, version, repo.get_latest_version(name) -def update_requirements(name, version=None, requirements_file='requirements.txt'): +def update_requirements(name, version=None, requirements_file="requirements.txt"): """ Upgrade or add `package_name` with `new_version` to the `requirements_file` requirements file. Write back requirements sorted with name and version @@ -2376,17 +2463,22 @@ def update_requirements(name, version=None, requirements_file='requirements.txt' if normalized_name == existing_normalized_name: if version != existing_version: is_updated = True - updated_name_versions.append((existing_normalized_name, existing_version,)) + updated_name_versions.append( + ( + existing_normalized_name, + existing_version, + ) + ) if is_updated: updated_name_versions = sorted(updated_name_versions) - nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions) + nvs = "\n".join(f"{name}=={version}" for name, version in updated_name_versions) - with open(requirements_file, 'w') as fo: + with open(requirements_file, "w") as fo: fo.write(nvs) -def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.txt'): +def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.txt"): """ Hash all the requirements found in the `requirements_file` requirements file based on distributions available in `dest_dir` @@ -2397,11 +2489,12 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.t for name, version in load_requirements(requirements_file, force_pinned=True): package = packages_by_normalized_name_version.get((name, version)) if not package: - raise Exception(f'Missing required package {name}=={version}') + raise Exception(f"Missing required package {name}=={version}") hashed.append(package.specifier_with_hashes) - with open(requirements_file, 'w') as fo: - fo.write('\n'.join(hashed)) + with open(requirements_file, "w") as fo: + fo.write("\n".join(hashed)) + ################################################################################ # @@ -2462,7 +2555,8 @@ def get_other_dists(_package, _dist): # try to get a latest version of the same package that is not our version other_local_packages = [ - p for p in local_repo.get_versions(local_package.name) + p + for p in local_repo.get_versions(local_package.name) if p.version != local_package.version ] @@ -2498,7 +2592,8 @@ def get_other_dists(_package, _dist): # try to get a latest version of the same package that is not our version other_remote_packages = [ - p for p in remote_repo.get_versions(local_package.name) + p + for p in remote_repo.get_versions(local_package.name) if p.version != local_package.version ] @@ -2534,10 +2629,11 @@ def get_other_dists(_package, _dist): # TODO: try to get data from dejacode if not local_dist.has_key_metadata(): - print(f'Unable to add essential ABOUT data for: {local_dist}') + print(f"Unable to add essential ABOUT data for: {local_dist}") if lic_errs: - lic_errs = '\n'.join(lic_errs) - print(f'Failed to fetch some licenses:: {lic_errs}') + lic_errs = "\n".join(lic_errs) + print(f"Failed to fetch some licenses:: {lic_errs}") + ################################################################################ # @@ -2551,19 +2647,18 @@ def call(args): Call args in a subprocess and display output on the fly. Return or raise stdout, stderr, returncode """ - if TRACE: print('Calling:', ' '.join(args)) + if TRACE: + print("Calling:", " ".join(args)) with subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding='utf-8' + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" ) as process: while True: line = process.stdout.readline() if not line and process.poll() is not None: break - if TRACE: print(line.rstrip(), flush=True) + if TRACE: + print(line.rstrip(), flush=True) stdout, stderr = process.communicate() returncode = process.returncode @@ -2595,15 +2690,17 @@ def add_or_upgrade_built_wheels( Include wheels for all dependencies if `with_deps` is True. Build remotely is `build_remotely` is True. """ - assert name, 'Name is required' - ver = version and f'=={version}' or '' - print(f'\nAdding wheels for package: {name}{ver}') + assert name, "Name is required" + ver = version and f"=={version}" or "" + print(f"\nAdding wheels for package: {name}{ver}") wheel_filenames = [] # a mapping of {req specifier: {mapping build_wheels kwargs}} wheels_to_build = {} for python_version, operating_system in itertools.product(python_versions, operating_systems): - print(f' Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}') + print( + f" Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}" + ) environment = Environment.from_pyver_and_os(python_version, operating_system) # Check if requested wheel already exists locally for this version @@ -2617,7 +2714,7 @@ def add_or_upgrade_built_wheels( wheel_filenames.append(wheel.filename) break if has_local_wheel: - print(f' local wheel exists: {wheel.filename}') + print(f" local wheel exists: {wheel.filename}") continue if not version: @@ -2626,17 +2723,18 @@ def add_or_upgrade_built_wheels( # Check if requested wheel already exists remotely or in Pypi for this version wheel_filename = fetch_package_wheel( - name=name, version=version, environment=environment, dest_dir=dest_dir) + name=name, version=version, environment=environment, dest_dir=dest_dir + ) if wheel_filename: wheel_filenames.append(wheel_filename) # the wheel is not available locally, remotely or in Pypi # we need to build binary from sources - requirements_specifier = f'{name}=={version}' + requirements_specifier = f"{name}=={version}" to_build = wheels_to_build.get(requirements_specifier) if to_build: - to_build['python_versions'].append(python_version) - to_build['operating_systems'].append(operating_system) + to_build["python_versions"].append(python_version) + to_build["operating_systems"].append(operating_system) else: wheels_to_build[requirements_specifier] = dict( requirements_specifier=requirements_specifier, @@ -2682,7 +2780,7 @@ def build_wheels( dest_dir=dest_dir, ) for local_build in builds: - print(f'Built wheel: {local_build}') + print(f"Built wheel: {local_build}") if all_pure: return builds @@ -2718,36 +2816,43 @@ def build_wheels_remotely_on_multiple_platforms( """ check_romp_is_configured() pyos_options = get_romp_pyos_options(python_versions, operating_systems) - deps = '' if with_deps else '--no-deps' - verbose = '--verbose' if verbose else '' - - romp_args = ([ - 'romp', - '--interpreter', 'cpython', - '--architecture', 'x86_64', - '--check-period', '5', # in seconds - - ] + pyos_options + [ - - '--artifact-paths', '*.whl', - '--artifact', 'artifacts.tar.gz', - '--command', + deps = "" if with_deps else "--no-deps" + verbose = "--verbose" if verbose else "" + + romp_args = ( + [ + "romp", + "--interpreter", + "cpython", + "--architecture", + "x86_64", + "--check-period", + "5", # in seconds + ] + + pyos_options + + [ + "--artifact-paths", + "*.whl", + "--artifact", + "artifacts.tar.gz", + "--command", # create a virtualenv, upgrade pip -# f'python -m ensurepip --user --upgrade; ' - f'python -m pip {verbose} install --user --upgrade pip setuptools wheel; ' - f'python -m pip {verbose} wheel {deps} {requirements_specifier}', - ]) + # f'python -m ensurepip --user --upgrade; ' + f"python -m pip {verbose} install --user --upgrade pip setuptools wheel; " + f"python -m pip {verbose} wheel {deps} {requirements_specifier}", + ] + ) if verbose: - romp_args.append('--verbose') + romp_args.append("--verbose") - print(f'Building wheels for: {requirements_specifier}') - print(f'Using command:', ' '.join(romp_args)) + print(f"Building wheels for: {requirements_specifier}") + print(f"Using command:", " ".join(romp_args)) call(romp_args) - wheel_filenames = extract_tar('artifacts.tar.gz', dest_dir) + wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir) for wfn in wheel_filenames: - print(f' built wheel: {wfn}') + print(f" built wheel: {wfn}") return wheel_filenames @@ -2763,12 +2868,16 @@ def get_romp_pyos_options( ... '--platform', 'windows'] >>> assert get_romp_pyos_options() == expected """ - python_dot_versions = ['.'.join(pv) for pv in sorted(set(python_versions))] - pyos_options = list(itertools.chain.from_iterable( - ('--version', ver) for ver in python_dot_versions)) + python_dot_versions = [".".join(pv) for pv in sorted(set(python_versions))] + pyos_options = list( + itertools.chain.from_iterable(("--version", ver) for ver in python_dot_versions) + ) - pyos_options += list(itertools.chain.from_iterable( - ('--platform' , plat) for plat in sorted(set(operating_systems)))) + pyos_options += list( + itertools.chain.from_iterable( + ("--platform", plat) for plat in sorted(set(operating_systems)) + ) + ) return pyos_options @@ -2776,17 +2885,18 @@ def get_romp_pyos_options( def check_romp_is_configured(): # these environment variable must be set before has_envt = ( - os.environ.get('ROMP_BUILD_REQUEST_URL') and - os.environ.get('ROMP_DEFINITION_ID') and - os.environ.get('ROMP_PERSONAL_ACCESS_TOKEN') and - os.environ.get('ROMP_USERNAME') + os.environ.get("ROMP_BUILD_REQUEST_URL") + and os.environ.get("ROMP_DEFINITION_ID") + and os.environ.get("ROMP_PERSONAL_ACCESS_TOKEN") + and os.environ.get("ROMP_USERNAME") ) if not has_envt: raise Exception( - 'ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, ' - 'ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME ' - 'are required enironment variables.') + "ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, " + "ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME " + "are required enironment variables." + ) def build_wheels_locally_if_pure_python( @@ -2804,19 +2914,24 @@ def build_wheels_locally_if_pure_python( Return a tuple of (True if all wheels are "pure", list of built wheel file names) """ - deps = [] if with_deps else ['--no-deps'] - verbose = ['--verbose'] if verbose else [] + deps = [] if with_deps else ["--no-deps"] + verbose = ["--verbose"] if verbose else [] - wheel_dir = tempfile.mkdtemp(prefix='scancode-release-wheels-local-') - cli_args = [ - 'pip', 'wheel', - '--wheel-dir', wheel_dir, - ] + deps + verbose + [ - requirements_specifier - ] + wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-local-") + cli_args = ( + [ + "pip", + "wheel", + "--wheel-dir", + wheel_dir, + ] + + deps + + verbose + + [requirements_specifier] + ) - print(f'Building local wheels for: {requirements_specifier}') - print(f'Using command:', ' '.join(cli_args)) + print(f"Building local wheels for: {requirements_specifier}") + print(f"Using command:", " ".join(cli_args)) call(cli_args) built = os.listdir(wheel_dir) @@ -2826,9 +2941,9 @@ def build_wheels_locally_if_pure_python( all_pure = all(is_pure_wheel(bwfn) for bwfn in built) if not all_pure: - print(f' Some wheels are not pure') + print(f" Some wheels are not pure") - print(f' Copying local wheels') + print(f" Copying local wheels") pure_built = [] for bwfn in built: owfn = os.path.join(dest_dir, bwfn) @@ -2836,7 +2951,7 @@ def build_wheels_locally_if_pure_python( nwfn = os.path.join(wheel_dir, bwfn) fileutils.copyfile(nwfn, owfn) pure_built.append(bwfn) - print(f' Built local wheel: {bwfn}') + print(f" Built local wheel: {bwfn}") return all_pure, pure_built @@ -2848,17 +2963,12 @@ def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): name of the new wheel if renamed or the existing new name otherwise. """ if is_pure_wheel(wheel_filename): - print(f'Pure wheel: {wheel_filename}, nothing to do.') + print(f"Pure wheel: {wheel_filename}, nothing to do.") return wheel_filename original_wheel_loc = os.path.join(dest_dir, wheel_filename) - wheel_dir = tempfile.mkdtemp(prefix='scancode-release-wheels-') - awargs = [ - 'auditwheel', - 'addtag', - '--wheel-dir', wheel_dir, - original_wheel_loc - ] + wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-") + awargs = ["auditwheel", "addtag", "--wheel-dir", wheel_dir, original_wheel_loc] call(awargs) audited = os.listdir(wheel_dir) @@ -2882,7 +2992,7 @@ def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): non_pypi_plats = utils_pypi_supported_tags.validate_platforms_for_pypi(new_wheel.platforms) new_wheel.platforms = [p for p in new_wheel.platforms if p not in non_pypi_plats] if not new_wheel.platforms: - print(f'Cannot make wheel PyPI compatible: {original_wheel_loc}') + print(f"Cannot make wheel PyPI compatible: {original_wheel_loc}") os.rename(new_wheel_loc, original_wheel_loc) return wheel_filename @@ -2892,18 +3002,20 @@ def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): return new_wheel_cleaned_filename -def extract_tar(location, dest_dir=THIRDPARTY_DIR,): +def extract_tar( + location, + dest_dir=THIRDPARTY_DIR, +): """ Extract a tar archive at `location` in the `dest_dir` directory. Return a list of extracted locations (either directories or files). """ - with open(location, 'rb') as fi: + with open(location, "rb") as fi: with tarfile.open(fileobj=fi) as tar: members = list(tar.getmembers()) tar.extractall(dest_dir, members=members) - return [os.path.basename(ti.name) for ti in members - if ti.type == tarfile.REGTYPE] + return [os.path.basename(ti.name) for ti in members if ti.type == tarfile.REGTYPE] def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): @@ -2918,25 +3030,23 @@ def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): wheel_filename = None remote_package = get_remote_package(name=name, version=version) if remote_package: - wheel_filename = remote_package.fetch_wheel( - environment=environment, dest_dir=dest_dir) + wheel_filename = remote_package.fetch_wheel(environment=environment, dest_dir=dest_dir) if wheel_filename: return wheel_filename pypi_package = get_pypi_package(name=name, version=version) if pypi_package: - wheel_filename = pypi_package.fetch_wheel( - environment=environment, dest_dir=dest_dir) + wheel_filename = pypi_package.fetch_wheel(environment=environment, dest_dir=dest_dir) return wheel_filename def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f'about check {dest_dir}'.split()) + subprocess.check_output(f"about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() - print('Invalid ABOUT files:') - print(cpe.output.decode('utf-8', errors='replace')) + print("Invalid ABOUT files:") + print(cpe.output.decode("utf-8", errors="replace")) def find_problems( @@ -2952,31 +3062,33 @@ def find_problems( for package in local_packages: if report_missing_sources and not package.sdist: - print(f'{package.name}=={package.version}: Missing source distribution.') + print(f"{package.name}=={package.version}: Missing source distribution.") if report_missing_wheels and not package.wheels: - print(f'{package.name}=={package.version}: Missing wheels.') + print(f"{package.name}=={package.version}: Missing wheels.") for dist in package.get_distributions(): dist.load_about_data(dest_dir=dest_dir) abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) if not dist.has_key_metadata(): - print(f' Missing key ABOUT data in file://{abpth}') - if 'classifiers' in dist.extra_data: - print(f' Dangling classifiers data in file://{abpth}') + print(f" Missing key ABOUT data in file://{abpth}") + if "classifiers" in dist.extra_data: + print(f" Dangling classifiers data in file://{abpth}") if not dist.validate_checksums(dest_dir): - print(f' Invalid checksums in file://{abpth}') + print(f" Invalid checksums in file://{abpth}") if not dist.sha1 and dist.md5: - print(f' Missing checksums in file://{abpth}') + print(f" Missing checksums in file://{abpth}") check_about(dest_dir=dest_dir) + def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return try: from packagedcode import pypi + return pypi.compute_normalized_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses lics = [python_safe_name(l).lower() for l in declared_licenses] - return ' AND '.join(lics).lower() + return " AND ".join(lics).lower() From 31ed4461437bfb5df3a4c4eef9e559de8d7a07e0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 18:39:18 +0200 Subject: [PATCH 09/62] Drop Ubuntu 16 add Python 3.10 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 22c12c4..b788ecb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,19 +7,11 @@ jobs: - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu16_cpython - image_name: ubuntu-16.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] - test_suites: - all: venv/bin/pytest -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu18_cpython image_name: ubuntu-18.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +19,7 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +27,7 @@ jobs: parameters: job_name: macos1014_cpython image_name: macos-10.14 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +35,7 @@ jobs: parameters: job_name: macos1015_cpython image_name: macos-10.15 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,7 +43,7 @@ jobs: parameters: job_name: win2016_cpython image_name: vs2017-win2016 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -59,6 +51,6 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 2ce7c7a1f1d4028d163de3a07a61b7c1febb9ae1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Oct 2021 19:14:08 +0200 Subject: [PATCH 10/62] Disable Python 3.10 tests on macOS 10.14 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b788ecb..7cd3025 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,7 +27,7 @@ jobs: parameters: job_name: macos1014_cpython image_name: macos-10.14 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.6', '3.7', '3.8', '3.9'] test_suites: all: venv/bin/pytest -n 2 -vvs From 2cc2c5a87dec0aa42e25c56f6110d461c44fb350 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 10 Nov 2021 15:14:18 +0800 Subject: [PATCH 11/62] Add code to remove the symlink before creating * It will prompts "Cannot create a file when that file already exists." if the symlink already exist. Signed-off-by: Chin Yeung Li --- configure.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure.bat b/configure.bat index 46ed4b3..4dfb201 100644 --- a/configure.bat +++ b/configure.bat @@ -160,6 +160,9 @@ if %ERRORLEVEL% neq 0 ( %CFG_REQUIREMENTS% @rem # Create junction to bin to have the same directory between linux and windows +if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( + rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" +) mklink /J %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts if %ERRORLEVEL% neq 0 ( From 6b2320aa10e79acdf92d5646ca6aec40725061cb Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Nov 2021 16:44:03 +0100 Subject: [PATCH 12/62] Improve handling licenses without scancode Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index e2778fe..c48484e 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -3089,6 +3089,5 @@ def compute_normalized_license_expression(declared_licenses): return pypi.compute_normalized_license(declared_licenses) except ImportError: - # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(l).lower() for l in declared_licenses] - return " AND ".join(lics).lower() + # Scancode is not installed, we join all license strings and return it + return " ".join(declared_licenses).lower() From 6ccff2b1312aa1246d0f27f6b36795b257d59813 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Nov 2021 19:08:47 +0100 Subject: [PATCH 13/62] Add support for deb and rpm containers Signed-off-by: Philippe Ombredanne --- etc/ci/azure-container-deb.yml | 50 ++++++ etc/ci/azure-container-rpm.yml | 51 ++++++ etc/ci/azure-posix.yml | 1 + etc/ci/install_sudo.sh | 15 ++ etc/ci/macports-ci | 304 +++++++++++++++++++++++++++++++++ etc/ci/macports-ci.ABOUT | 16 ++ etc/ci/mit.LICENSE | 5 + 7 files changed, 442 insertions(+) create mode 100644 etc/ci/azure-container-deb.yml create mode 100644 etc/ci/azure-container-rpm.yml create mode 100644 etc/ci/install_sudo.sh create mode 100644 etc/ci/macports-ci create mode 100644 etc/ci/macports-ci.ABOUT create mode 100644 etc/ci/mit.LICENSE diff --git a/etc/ci/azure-container-deb.yml b/etc/ci/azure-container-deb.yml new file mode 100644 index 0000000..85b611d --- /dev/null +++ b/etc/ci/azure-container-deb.yml @@ -0,0 +1,50 @@ +parameters: + job_name: '' + container: '' + python_path: '' + python_version: '' + package_manager: apt-get + install_python: '' + install_packages: | + set -e -x + sudo apt-get -y update + sudo apt-get -y install \ + build-essential \ + xz-utils zlib1g bzip2 libbz2-1.0 tar \ + sqlite3 libxml2-dev libxslt1-dev \ + software-properties-common openssl + test_suite: '' + test_suite_label: '' + + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: 'ubuntu-16.04' + + container: + image: ${{ parameters.container }} + options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' + + steps: + - checkout: self + fetchDepth: 10 + + - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} + displayName: Install sudo + + - script: ${{ parameters.install_packages }} + displayName: Install required packages + + - script: ${{ parameters.install_python }} + displayName: 'Install Python ${{ parameters.python_version }}' + + - script: ${{ parameters.python_path }} --version + displayName: 'Show Python version' + + - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev + displayName: 'Run Configure' + + - script: ${{ parameters.test_suite }} + displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-container-rpm.yml b/etc/ci/azure-container-rpm.yml new file mode 100644 index 0000000..1e6657d --- /dev/null +++ b/etc/ci/azure-container-rpm.yml @@ -0,0 +1,51 @@ +parameters: + job_name: '' + image_name: 'ubuntu-16.04' + container: '' + python_path: '' + python_version: '' + package_manager: yum + install_python: '' + install_packages: | + set -e -x + sudo yum groupinstall -y "Development Tools" + sudo yum install -y \ + openssl openssl-devel \ + sqlite-devel zlib-devel xz-devel bzip2-devel \ + bzip2 tar unzip zip \ + libxml2-devel libxslt-devel + test_suite: '' + test_suite_label: '' + + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + container: + image: ${{ parameters.container }} + options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' + + steps: + - checkout: self + fetchDepth: 10 + + - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} + displayName: Install sudo + + - script: ${{ parameters.install_packages }} + displayName: Install required packages + + - script: ${{ parameters.install_python }} + displayName: 'Install Python ${{ parameters.python_version }}' + + - script: ${{ parameters.python_path }} --version + displayName: 'Show Python version' + + - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev + displayName: 'Run Configure' + + - script: ${{ parameters.test_suite }} + displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 0921d9b..7a9acff 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -31,6 +31,7 @@ jobs: displayName: 'Install Python $(python_version)' - script: | + python --version python3 --version python$(python_version) --version echo "python$(python_version)" > PYTHON_EXECUTABLE diff --git a/etc/ci/install_sudo.sh b/etc/ci/install_sudo.sh new file mode 100644 index 0000000..77f4210 --- /dev/null +++ b/etc/ci/install_sudo.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + + +if [[ "$1" == "apt-get" ]]; then + apt-get update -y + apt-get -o DPkg::Options::="--force-confold" install -y sudo + +elif [[ "$1" == "yum" ]]; then + yum install -y sudo + +elif [[ "$1" == "dnf" ]]; then + dnf install -y sudo + +fi diff --git a/etc/ci/macports-ci b/etc/ci/macports-ci new file mode 100644 index 0000000..ac474e4 --- /dev/null +++ b/etc/ci/macports-ci @@ -0,0 +1,304 @@ +#! /bin/bash + +# Copyright (c) 2019 Giovanni Bussi + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +export COLUMNS=80 + +if [ "$GITHUB_ACTIONS" = true ] ; then + echo "COLUMNS=$COLUMNS" >> "$GITHUB_ENV" +fi + +# file to be source at the end of subshell: +export MACPORTS_CI_SOURCEME="$(mktemp)" + +( +# start subshell +# this allows to use the script in two ways: +# 1. as ./macports-ci +# 2. as source ./macports-ci +# as of now, choice 2 only changes the env var COLUMNS. + +MACPORTS_VERSION=2.6.4 +MACPORTS_PREFIX=/opt/local +MACPORTS_SYNC=tarball + +action=$1 +shift + +case "$action" in +(install) + +echo "macports-ci: install" + +KEEP_BREW=yes + +for opt +do + case "$opt" in + (--source) SOURCE=yes ;; + (--binary) SOURCE=no ;; + (--keep-brew) KEEP_BREW=yes ;; + (--remove-brew) KEEP_BREW=no ;; + (--version=*) MACPORTS_VERSION="${opt#--version=}" ;; + (--prefix=*) MACPORTS_PREFIX="${opt#--prefix=}" ;; + (--sync=*) MACPORTS_SYNC="${opt#--sync=}" ;; + (*) echo "macports-ci: unknown option $opt" + exit 1 ;; + esac +done + +if test "$KEEP_BREW" = no ; then + echo "macports-ci: removing homebrew" + pushd "$(mktemp -d)" + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > uninstall + chmod +x uninstall + ./uninstall --force + popd +else + echo "macports-ci: keeping HomeBrew" +fi + +echo "macports-ci: prefix=$MACPORTS_PREFIX" + +if test "$MACPORTS_PREFIX" != /opt/local ; then + echo "macports-ci: Installing on non standard prefix $MACPORTS_PREFIX can be only made from sources" + SOURCE=yes +fi + +if test "$SOURCE" = yes ; then + echo "macports-ci: Installing from source" +else + echo "macports-ci: Installing from binary" +fi + +echo "macports-ci: Sync mode=$MACPORTS_SYNC" + +pushd "$(mktemp -d)" + +OSX_VERSION="$(sw_vers -productVersion | grep -o '^[0-9][0-9]*\.[0-9][0-9]*')" + +if test "$OSX_VERSION" == 10.10 ; then + OSX_NAME=Yosemite +elif test "$OSX_VERSION" == 10.11 ; then + OSX_NAME=ElCapitan +elif test "$OSX_VERSION" == 10.12 ; then + OSX_NAME=Sierra +elif test "$OSX_VERSION" == 10.13 ; then + OSX_NAME=HighSierra +elif test "$OSX_VERSION" == 10.14 ; then + OSX_NAME=Mojave +elif test "$OSX_VERSION" == 10.15 ; then + OSX_NAME=Catalina +else + echo "macports-ci: Unknown OSX version $OSX_VERSION" + exit 1 +fi + +echo "macports-ci: OSX version $OSX_VERSION $OSX_NAME" + +MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg + +# this is a workaround needed because binary installer MacPorts-2.6.3-10.12-Sierra.pkg is broken +if [ "$SOURCE" != yes ] && [ "$MACPORTS_PKG" = "MacPorts-2.6.3-10.12-Sierra.pkg" ] ; then + echo "macports-ci: WARNING $MACPORTS_PKG installer is broken" + echo "macports-ci: reverting to 2.6.2 installer followed by selfupdate" + MACPORTS_VERSION=2.6.2 + MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg +fi + +URL="https://distfiles.macports.org/MacPorts" +URL="https://github.com/macports/macports-base/releases/download/v$MACPORTS_VERSION/" + +echo "macports-ci: Base URL is $URL" + +if test "$SOURCE" = yes ; then +# download source: + curl -LO $URL/MacPorts-${MACPORTS_VERSION}.tar.bz2 + tar xjf MacPorts-${MACPORTS_VERSION}.tar.bz2 + cd MacPorts-${MACPORTS_VERSION} +# install + ./configure --prefix="$MACPORTS_PREFIX" --with-applications-dir="$MACPORTS_PREFIX/Applications" >/dev/null && + sudo make install >/dev/null +else + +# download installer: + curl -LO $URL/$MACPORTS_PKG +# install: + sudo installer -verbose -pkg $MACPORTS_PKG -target / +fi + +# update: +export PATH="$MACPORTS_PREFIX/bin:$PATH" + +echo "PATH=\"$MACPORTS_PREFIX/bin:\$PATH\"" > "$MACPORTS_CI_SOURCEME" + +if [ "$GITHUB_ACTIONS" = true ] ; then + echo "$MACPORTS_PREFIX/bin" >> "$GITHUB_PATH" +fi + + +SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf + +case "$MACPORTS_SYNC" in +(rsync) + echo "macports-ci: Using rsync" + ;; +(github) + echo "macports-ci: Using github" + pushd "$MACPORTS_PREFIX"/var/macports/sources + sudo mkdir -p github.com/macports/macports-ports/ + sudo chown -R $USER:admin github.com + git clone https://github.com/macports/macports-ports.git github.com/macports/macports-ports/ + awk '{if($NF=="[default]") print "file:///opt/local/var/macports/sources/github.com/macports/macports-ports/"; else print}' "$SOURCES" > $HOME/$$.tmp + sudo mv -f $HOME/$$.tmp "$SOURCES" + popd + ;; +(tarball) + echo "macports-ci: Using tarball" + awk '{if($NF=="[default]") print "https://distfiles.macports.org/ports.tar.gz [default]"; else print}' "$SOURCES" > $$.tmp + sudo mv -f $$.tmp "$SOURCES" + ;; +(*) + echo "macports-ci: Unknown sync mode $MACPORTS_SYNC" + ;; +esac + +i=1 +# run through a while to retry upon failure +while true +do + echo "macports-ci: Trying to selfupdate (iteration $i)" +# here I test for the presence of a known portfile +# this check confirms that ports were installed +# notice that port -N selfupdate && break is not sufficient as a test +# (sometime it returns a success even though ports have not been installed) +# for some misterious reasons, running without "-d" does not work in some case + sudo port -d -N selfupdate 2>&1 | grep -v DEBUG | awk '{if($1!="x")print}' + port info xdrfile > /dev/null && break || true + sleep 5 + i=$((i+1)) + if ((i>20)) ; then + echo "macports-ci: Failed after $i iterations" + exit 1 + fi +done + +echo "macports-ci: Selfupdate successful after $i iterations" + +dir="$PWD" +popd +sudo rm -fr $dir + +;; + +(localports) + +echo "macports-ci: localports" + +for opt +do + case "$opt" in + (*) ports="$opt" ;; + esac +done + +if ! test -d "$ports" ; then + echo "macports-ci: Please provide a port directory" + exit 1 +fi + +w=$(which port) + +MACPORTS_PREFIX="${w%/bin/port}" + +cd "$ports" + +ports="$(pwd)" + +echo "macports-ci: Portdir fullpath: $ports" +SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf + +awk -v repo="file://$ports" '{if($NF=="[default]") print repo; print}' "$SOURCES" > $$.tmp +sudo mv -f $$.tmp "$SOURCES" + +portindex + +;; + +(ccache) +w=$(which port) +MACPORTS_PREFIX="${w%/bin/port}" + +echo "macports-ci: ccache" + +ccache_do=install + +for opt +do + case "$opt" in + (--save) ccache_do=save ;; + (--install) ccache_do=install ;; + (*) echo "macports-ci: ccache: unknown option $opt" + exit 1 ;; + esac +done + + +case "$ccache_do" in +(install) +# first install ccache +sudo port -N install ccache +# then tell macports to use it +CONF="${MACPORTS_PREFIX}"/etc/macports/macports.conf +awk '{if(match($0,"configureccache")) print "configureccache yes" ; else print }' "$CONF" > $$.tmp +sudo mv -f $$.tmp "$CONF" + +# notice that cache size is set to 512Mb, same as it is set by Travis-CI on linux +# might be changed in the future +test -f "$HOME"/.macports-ci-ccache/ccache.conf && + sudo rm -fr "$MACPORTS_PREFIX"/var/macports/build/.ccache && + sudo mkdir -p "$MACPORTS_PREFIX"/var/macports/build/.ccache && + sudo cp -a "$HOME"/.macports-ci-ccache/* "$MACPORTS_PREFIX"/var/macports/build/.ccache/ && + sudo echo "max_size = 512M" > "$MACPORTS_PREFIX"/var/macports/build/.ccache/ccache.conf && + sudo chown -R macports:admin "$MACPORTS_PREFIX"/var/macports/build/.ccache + +;; +(save) + +sudo rm -fr "$HOME"/.macports-ci-ccache +sudo mkdir -p "$HOME"/.macports-ci-ccache +sudo cp -a "$MACPORTS_PREFIX"/var/macports/build/.ccache/* "$HOME"/.macports-ci-ccache/ + +esac + +CCACHE_DIR="$MACPORTS_PREFIX"/var/macports/build/.ccache/ ccache -s + +;; + +(*) +echo "macports-ci: unknown action $action" + +esac + +) + +# allows setting env var if necessary: +source "$MACPORTS_CI_SOURCEME" diff --git a/etc/ci/macports-ci.ABOUT b/etc/ci/macports-ci.ABOUT new file mode 100644 index 0000000..60a11f8 --- /dev/null +++ b/etc/ci/macports-ci.ABOUT @@ -0,0 +1,16 @@ +about_resource: macports-ci +name: macports-ci +version: c9676e67351a3a519e37437e196cd0ee9c2180b8 +download_url: https://raw.githubusercontent.com/GiovanniBussi/macports-ci/c9676e67351a3a519e37437e196cd0ee9c2180b8/macports-ci +description: Simplify MacPorts setup on Travis-CI +homepage_url: https://github.com/GiovanniBussi/macports-ci +license_expression: mit +copyright: Copyright (c) Giovanni Bussi +attribute: yes +checksum_md5: 5d31d479132502f80acdaed78bed9e23 +checksum_sha1: 74b15643bd1a528d91b4a7c2169c6fc656f549c2 +package_url: pkg:github/giovannibussi/macports-ci@c9676e67351a3a519e37437e196cd0ee9c2180b8#macports-ci +licenses: + - key: mit + name: MIT License + file: mit.LICENSE diff --git a/etc/ci/mit.LICENSE b/etc/ci/mit.LICENSE new file mode 100644 index 0000000..e662c78 --- /dev/null +++ b/etc/ci/mit.LICENSE @@ -0,0 +1,5 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file From 6962f8bbe9c40e8dcacab0ab3325c0bc882e9a4a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 26 Nov 2021 19:09:35 +0100 Subject: [PATCH 14/62] Support licenses when ScanCode is not installed Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index c48484e..e2778fe 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -3089,5 +3089,6 @@ def compute_normalized_license_expression(declared_licenses): return pypi.compute_normalized_license(declared_licenses) except ImportError: - # Scancode is not installed, we join all license strings and return it - return " ".join(declared_licenses).lower() + # Scancode is not installed, clean and join all the licenses + lics = [python_safe_name(l).lower() for l in declared_licenses] + return " AND ".join(lics).lower() From 2f77f979c9b83e5365350405c1c60fe08cb75c10 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 12 Jan 2022 09:20:40 +0100 Subject: [PATCH 15/62] Improve wheel build Allow to launch builds and then fetch built wheels later Improve support for newer Pythons and OS versions. Signed-off-by: Philippe Ombredanne --- etc/scripts/build_wheels.py | 12 ++ etc/scripts/fetch_built_wheels.py | 33 ++++ etc/scripts/fix_thirdparty.py | 15 +- .../test_utils_pip_compatibility_tags.py | 6 +- etc/scripts/utils_thirdparty.py | 181 ++++++++++++++---- 5 files changed, 204 insertions(+), 43 deletions(-) create mode 100644 etc/scripts/fetch_built_wheels.py diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py index 5a39c78..8a28176 100644 --- a/etc/scripts/build_wheels.py +++ b/etc/scripts/build_wheels.py @@ -69,6 +69,16 @@ is_flag=True, help="Also include all dependent wheels.", ) +@click.option( + "--remote-build-log-file", + type=click.Path(writable=True), + default=None, + metavar="LOG-FILE", + help="Path to an optional log file where to list remote builds download URLs. " + "If provided, do not wait for remote builds to complete (and therefore, " + "do not download them either). Instead create a JSON lines log file with " + "one entry for each build suitable to fetch the artifacts at a later time.", +) @click.option( "--verbose", is_flag=True, @@ -83,6 +93,7 @@ def build_wheels( operating_system, with_deps, build_remotely, + remote_build_log_file, verbose, ): """ @@ -102,6 +113,7 @@ def build_wheels( build_remotely=build_remotely, with_deps=with_deps, verbose=verbose, + remote_build_log_file=remote_build_log_file, ) diff --git a/etc/scripts/fetch_built_wheels.py b/etc/scripts/fetch_built_wheels.py new file mode 100644 index 0000000..4fea16c --- /dev/null +++ b/etc/scripts/fetch_built_wheels.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import click + +import utils_thirdparty + + +@click.command() +@click.option( + "-d", + "--thirdparty-dir", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + required=True, + help="Path to the thirdparty directory to check.", +) +@click.help_option("-h", "--help") +def check_thirdparty_dir(thirdparty_dir): + """ + Check a thirdparty directory for problems. + """ + utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + + +if __name__ == "__main__": + check_thirdparty_dir() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py index 9b1cbc4..c14b7d5 100644 --- a/etc/scripts/fix_thirdparty.py +++ b/etc/scripts/fix_thirdparty.py @@ -31,11 +31,22 @@ is_flag=True, help="Build missing wheels remotely.", ) +@click.option( + "--remote-build-log-file", + type=click.Path(writable=True), + default=None, + metavar="LOG-FILE", + help="Path to an optional log file where to list remote builds download URLs. " + "If provided, do not wait for remote builds to complete (and therefore, " + "do not download them either). Instead create a JSON lines log file with " + "one entry for each build suitable to fetch the artifacts at a later time.", +) @click.help_option("-h", "--help") def fix_thirdparty_dir( thirdparty_dir, build_wheels, build_remotely, + remote_build_log_file, ): """ Fix a thirdparty directory of dependent package wheels and sdist. @@ -58,11 +69,13 @@ def fix_thirdparty_dir( package_envts_not_built = [] if build_wheels: print("***BUILD*** MISSING WHEELS") - package_envts_not_built, _wheel_filenames_built = utils_thirdparty.build_missing_wheels( + results = utils_thirdparty.build_missing_wheels( packages_and_envts=package_envts_not_fetched, build_remotely=build_remotely, + remote_build_log_file=remote_build_log_file, dest_dir=thirdparty_dir, ) + package_envts_not_built, _wheel_filenames_built = results print("***ADD*** ABOUT AND LICENSES") utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index 722fa70..98187c5 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -47,7 +47,7 @@ ], ) def test_version_info_to_nodot(version_info, expected): - actual = pip_compatibility_tags.version_info_to_nodot(version_info) + actual = utils_pip_compatibility_tags.version_info_to_nodot(version_info) assert actual == expected @@ -95,7 +95,7 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): Specifying manylinux2010 implies manylinux1. """ groups = {} - supported = pip_compatibility_tags.get_supported(platforms=[manylinux2010]) + supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinux2010]) for tag in supported: groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) @@ -118,7 +118,7 @@ def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): Specifying manylinux2014 implies manylinux2010/manylinux1. """ groups = {} - supported = pip_compatibility_tags.get_supported(platforms=[manylinuxA]) + supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinuxA]) for tag in supported: groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index e2778fe..d25f0c2 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -89,21 +89,39 @@ # Supported environments PYTHON_VERSIONS = "36", "37", "38", "39", "310" +PYTHON_DOT_VERSIONS_BY_VER = { + "36": "3.6", + "37": "3.7", + "38": "3.8", + "39": "3.9", + "310": "3.10", +} + +def get_python_dot_version(version): + """ + Return a dot version from a plain, non-dot version. + """ + return PYTHON_DOT_VERSIONS_BY_VER[version] + ABIS_BY_PYTHON_VERSION = { "36": ["cp36", "cp36m"], "37": ["cp37", "cp37m"], "38": ["cp38", "cp38m"], "39": ["cp39", "cp39m"], "310": ["cp310", "cp310m"], + "36": ["cp36", "abi3"], + "37": ["cp37", "abi3"], + "38": ["cp38", "abi3"], + "39": ["cp39", "abi3"], + "310": ["cp310", "abi3"], } PLATFORMS_BY_OS = { "linux": [ "linux_x86_64", "manylinux1_x86_64", - "manylinux2014_x86_64", "manylinux2010_x86_64", - "manylinux_2_12_x86_64", + "manylinux2014_x86_64", ], "macos": [ "macosx_10_6_intel", @@ -122,8 +140,8 @@ "macosx_10_14_x86_64", "macosx_10_15_intel", "macosx_10_15_x86_64", - "macosx_10_15_x86_64", "macosx_11_0_x86_64", + "macosx_11_intel", # 'macosx_11_0_arm64', ], "windows": [ @@ -157,6 +175,10 @@ LICENSING = license_expression.Licensing() +# time to wait build for in seconds, as a string +# 0 measn no wait +DEFAULT_ROMP_BUILD_WAIT = "5" + ################################################################################ # # Fetch remote wheels and sources locally @@ -1478,7 +1500,8 @@ def fetch_wheel( else: fetched_filenames = set() - for wheel in self.get_supported_wheels(environment): + supported_wheels = list(self.get_supported_wheels(environment)) + for wheel in supported_wheels: if wheel.filename not in fetched_filenames: fetch_and_save_path_or_url( @@ -2212,6 +2235,7 @@ def build_missing_wheels( build_remotely=False, with_deps=False, dest_dir=THIRDPARTY_DIR, + remote_build_log_file=None, ): """ Build all wheels in a list of tuple (Package, Environment) and save in @@ -2237,8 +2261,9 @@ def build_missing_wheels( build_remotely=build_remotely, python_versions=python_versions, operating_systems=operating_systems, - verbose=False, + verbose=TRACE, dest_dir=dest_dir, + remote_build_log_file=remote_build_log_file, ) print(".") except Exception as e: @@ -2642,26 +2667,32 @@ def get_other_dists(_package, _dist): ################################################################################ -def call(args): +def call(args, verbose=TRACE): """ - Call args in a subprocess and display output on the fly. + Call args in a subprocess and display output on the fly if ``trace`` is True. Return or raise stdout, stderr, returncode """ if TRACE: print("Calling:", " ".join(args)) with subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8" + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: + stdouts = [] while True: line = process.stdout.readline() if not line and process.poll() is not None: break - if TRACE: + stdouts.append(line) + if verbose: print(line.rstrip(), flush=True) stdout, stderr = process.communicate() + if not stdout.strip(): + stdout = "\n".join(stdouts) + returncode = process.returncode + if returncode == 0: return returncode, stdout, stderr else: @@ -2676,7 +2707,8 @@ def add_or_upgrade_built_wheels( dest_dir=THIRDPARTY_DIR, build_remotely=False, with_deps=False, - verbose=False, + verbose=TRACE, + remote_build_log_file=None, ): """ Add or update package `name` and `version` as a binary wheel saved in @@ -2689,11 +2721,17 @@ def add_or_upgrade_built_wheels( Include wheels for all dependencies if `with_deps` is True. Build remotely is `build_remotely` is True. + Do not wait for build completion and log to ``remote_build_log_file`` + file path if provided. """ assert name, "Name is required" ver = version and f"=={version}" or "" print(f"\nAdding wheels for package: {name}{ver}") + if verbose: + print("python_versions:", python_versions) + print("operating_systems:", operating_systems) + wheel_filenames = [] # a mapping of {req specifier: {mapping build_wheels kwargs}} wheels_to_build = {} @@ -2725,6 +2763,8 @@ def add_or_upgrade_built_wheels( wheel_filename = fetch_package_wheel( name=name, version=version, environment=environment, dest_dir=dest_dir ) + if verbose: + print(" fetching package wheel:", wheel_filename) if wheel_filename: wheel_filenames.append(wheel_filename) @@ -2744,6 +2784,7 @@ def add_or_upgrade_built_wheels( build_remotely=build_remotely, with_deps=with_deps, verbose=verbose, + remote_build_log_file=remote_build_log_file, ) for build_wheels_kwargs in wheels_to_build.values(): @@ -2761,6 +2802,7 @@ def build_wheels( build_remotely=False, with_deps=False, verbose=False, + remote_build_log_file=None, ): """ Given a pip `requirements_specifier` string (such as package names or as @@ -2772,6 +2814,9 @@ def build_wheels( First try to build locally to process pure Python wheels, and fall back to build remotey on all requested Pythons and operating systems. + + Do not wait for build completion and log to ``remote_build_log_file`` + file path if provided. """ all_pure, builds = build_wheels_locally_if_pure_python( requirements_specifier=requirements_specifier, @@ -2793,6 +2838,7 @@ def build_wheels( operating_systems=operating_systems, verbose=verbose, dest_dir=dest_dir, + remote_build_log_file=remote_build_log_file, ) builds.extend(remote_builds) @@ -2806,6 +2852,7 @@ def build_wheels_remotely_on_multiple_platforms( operating_systems=PLATFORMS_BY_OS, verbose=False, dest_dir=THIRDPARTY_DIR, + remote_build_log_file=None, ): """ Given pip `requirements_specifier` string (such as package names or as @@ -2813,35 +2860,43 @@ def build_wheels_remotely_on_multiple_platforms( all dependencies for all `python_versions` and `operating_systems` combinations and save them back in `dest_dir` and return a list of built wheel file names. + + Do not wait for build completion and log to ``remote_build_log_file`` + file path if provided. """ check_romp_is_configured() pyos_options = get_romp_pyos_options(python_versions, operating_systems) deps = "" if with_deps else "--no-deps" verbose = "--verbose" if verbose else "" - romp_args = ( - [ - "romp", - "--interpreter", - "cpython", - "--architecture", - "x86_64", - "--check-period", - "5", # in seconds - ] - + pyos_options - + [ - "--artifact-paths", - "*.whl", - "--artifact", - "artifacts.tar.gz", - "--command", - # create a virtualenv, upgrade pip - # f'python -m ensurepip --user --upgrade; ' - f"python -m pip {verbose} install --user --upgrade pip setuptools wheel; " - f"python -m pip {verbose} wheel {deps} {requirements_specifier}", - ] - ) + if remote_build_log_file: + # zero seconds, no wait, log to file instead + wait_build_for = "0" + else: + wait_build_for = DEFAULT_ROMP_BUILD_WAIT + + romp_args = [ + "romp", + "--interpreter", + "cpython", + "--architecture", + "x86_64", + "--check-period", + wait_build_for, # in seconds + ] + + if remote_build_log_file: + romp_args += ["--build-log-file", remote_build_log_file] + + romp_args += pyos_options + [ + "--artifact-paths", + "*.whl", + "--artifact", + "artifacts.tar.gz", + "--command", + f"python -m pip {verbose} install --user --upgrade pip setuptools wheel; " + f"python -m pip {verbose} wheel {deps} {requirements_specifier}", + ] if verbose: romp_args.append("--verbose") @@ -2849,10 +2904,54 @@ def build_wheels_remotely_on_multiple_platforms( print(f"Building wheels for: {requirements_specifier}") print(f"Using command:", " ".join(romp_args)) call(romp_args) + wheel_filenames = [] + if not remote_build_log_file: + wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir) + for wfn in wheel_filenames: + print(f" built wheel: {wfn}") + return wheel_filenames + + +def fetch_remotely_built_wheels( + remote_build_log_file, + dest_dir=THIRDPARTY_DIR, + no_wait=False, + verbose=False, +): + """ + Given a ``remote_build_log_file`` file path with a JSON lines log of a + remote build, fetch the built wheels and move them to ``dest_dir``. Return a + list of built wheel file names. + """ + wait = "0" if no_wait else DEFAULT_ROMP_BUILD_WAIT # in seconds + + romp_args = [ + "romp-fetch", + "--build-log-file", + remote_build_log_file, + "--check-period", + wait, + ] - wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir) - for wfn in wheel_filenames: - print(f" built wheel: {wfn}") + if verbose: + romp_args.append("--verbose") + + print(f"Fetching built wheels from log file: {remote_build_log_file}") + print(f"Using command:", " ".join(romp_args)) + + call(romp_args, verbose=verbose) + + wheel_filenames = [] + + for art in os.listdir(os.getcwd()): + if not art.endswith("artifacts.tar.gz") or not os.path.getsize(art): + continue + + print(f" Processing artifact archive: {art}") + wheel_fns = extract_tar(art, dest_dir) + for wfn in wheel_fns: + print(f" Retrieved built wheel: {wfn}") + wheel_filenames.extend(wheel_fns) return wheel_filenames @@ -2864,11 +2963,11 @@ def get_romp_pyos_options( Return a list of CLI options for romp For example: >>> expected = ['--version', '3.6', '--version', '3.7', '--version', '3.8', - ... '--version', '3.9', '--platform', 'linux', '--platform', 'macos', - ... '--platform', 'windows'] + ... '--version', '3.9', '--version', '3.10', '--platform', 'linux', + ... '--platform', 'macos', '--platform', 'windows'] >>> assert get_romp_pyos_options() == expected """ - python_dot_versions = [".".join(pv) for pv in sorted(set(python_versions))] + python_dot_versions = [get_python_dot_version(pv) for pv in sorted(set(python_versions))] pyos_options = list( itertools.chain.from_iterable(("--version", ver) for ver in python_dot_versions) ) @@ -3029,12 +3128,16 @@ def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): """ wheel_filename = None remote_package = get_remote_package(name=name, version=version) + if TRACE: + print(" remote_package:", remote_package) if remote_package: wheel_filename = remote_package.fetch_wheel(environment=environment, dest_dir=dest_dir) if wheel_filename: return wheel_filename pypi_package = get_pypi_package(name=name, version=version) + if TRACE: + print(" pypi_package:", pypi_package) if pypi_package: wheel_filename = pypi_package.fetch_wheel(environment=environment, dest_dir=dest_dir) return wheel_filename From 784e701e7d266c1e20ee304abfa59ebe843b53b5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 12 Jan 2022 13:02:39 +0100 Subject: [PATCH 16/62] Aligne with latest ScanCode TK updates Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 20 +++++++++++++++++--- pyproject.toml | 5 ++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index b792d9f..74b8649 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,8 +18,8 @@ # -- Project information ----------------------------------------------------- project = "nexb-skeleton" -copyright = "nexb Inc." -author = "nexb Inc." +copyright = "nexB Inc. and others." +author = "AboutCode.org authors and contributors" # -- General configuration --------------------------------------------------- @@ -27,7 +27,19 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = [ +'sphinx.ext.intersphinx', +] + +# This points to aboutcode.readthedocs.io +# In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot +# Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 + +intersphinx_mapping = { + 'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None), + 'scancode-workbench': ('https://scancode-workbench.readthedocs.io/en/develop/', None), +} + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -50,6 +62,8 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] +master_doc = 'index' + html_context = { "css_files": [ "_static/theme_overrides.css", # override wide tables in RTD theme diff --git a/pyproject.toml b/pyproject.toml index 1e10f32..5ebaa03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,9 @@ norecursedirs = [ "tmp", "venv", "tests/data", - ".eggs" + ".eggs", + "src/*/data", + "tests/*/data" ] python_files = "*.py" @@ -46,5 +48,6 @@ python_functions = "test" addopts = [ "-rfExXw", "--strict-markers", + "--ignore setup.py", "--doctest-modules" ] From a7c2efdeca1bdb5595484af0c7c5bef55f451c85 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 12 Jan 2022 14:07:31 +0100 Subject: [PATCH 17/62] Do not ignore setup.py Signed-off-by: Philippe Ombredanne --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5ebaa03..cde7907 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,5 @@ python_functions = "test" addopts = [ "-rfExXw", "--strict-markers", - "--ignore setup.py", "--doctest-modules" ] From b1dabd894c469174b0fee950043f7d29fac6027f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 12 Jan 2022 18:17:16 +0100 Subject: [PATCH 18/62] Update scripts Make fetch_built_wheel work Add new strip classifiers option to fix_thirdparty Improve simple requirements parsing to get the latest versions Signed-off-by: Philippe Ombredanne --- etc/scripts/fetch_built_wheels.py | 44 +++++++++++++++++---- etc/scripts/fix_thirdparty.py | 66 +++++++++++++++++++------------ etc/scripts/utils_requirements.py | 59 +++++++++++++++++++++++---- etc/scripts/utils_thirdparty.py | 10 ++++- 4 files changed, 138 insertions(+), 41 deletions(-) diff --git a/etc/scripts/fetch_built_wheels.py b/etc/scripts/fetch_built_wheels.py index 4fea16c..a78861e 100644 --- a/etc/scripts/fetch_built_wheels.py +++ b/etc/scripts/fetch_built_wheels.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/nexB/scancode-toolkit for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click @@ -14,20 +14,50 @@ @click.command() +@click.option( + "--remote-build-log-file", + type=click.Path(readable=True), + metavar="LOG-FILE", + help="Path to a remote builds log file.", +) @click.option( "-d", "--thirdparty-dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - required=True, - help="Path to the thirdparty directory to check.", + metavar="DIR", + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help="Path to the thirdparty directory to save built wheels.", +) +@click.option( + "--no-wait", + is_flag=True, + default=False, + help="Do not wait for build completion.", +) +@click.option( + "--verbose", + is_flag=True, + help="Provide verbose output.", ) @click.help_option("-h", "--help") -def check_thirdparty_dir(thirdparty_dir): +def fetch_remote_wheels( + remote_build_log_file, + thirdparty_dir, + no_wait, + verbose, +): """ - Check a thirdparty directory for problems. + Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines + build log file. """ - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + utils_thirdparty.fetch_remotely_built_wheels( + remote_build_log_file=remote_build_log_file, + dest_dir=thirdparty_dir, + no_wait=no_wait, + verbose=verbose, + ) if __name__ == "__main__": - check_thirdparty_dir() + fetch_remote_wheels() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py index c14b7d5..9d401cd 100644 --- a/etc/scripts/fix_thirdparty.py +++ b/etc/scripts/fix_thirdparty.py @@ -41,12 +41,18 @@ "do not download them either). Instead create a JSON lines log file with " "one entry for each build suitable to fetch the artifacts at a later time.", ) +@click.option( + "--strip-classifiers", + is_flag=True, + help="Remove danglingf classifiers", +) @click.help_option("-h", "--help") def fix_thirdparty_dir( thirdparty_dir, build_wheels, build_remotely, remote_build_log_file, + strip_classifiers, ): """ Fix a thirdparty directory of dependent package wheels and sdist. @@ -61,35 +67,45 @@ def fix_thirdparty_dir( Optionally build missing binary wheels for all supported OS and Python version combos locally or remotely. """ - print("***FETCH*** MISSING WHEELS") - package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) - print("***FETCH*** MISSING SOURCES") - src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - package_envts_not_built = [] - if build_wheels: - print("***BUILD*** MISSING WHEELS") - results = utils_thirdparty.build_missing_wheels( - packages_and_envts=package_envts_not_fetched, - build_remotely=build_remotely, - remote_build_log_file=remote_build_log_file, + if strip_classifiers: + print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS") + utils_thirdparty.add_fetch_or_update_about_and_license_files( dest_dir=thirdparty_dir, + strip_classifiers=strip_classifiers, ) - package_envts_not_built, _wheel_filenames_built = results - - print("***ADD*** ABOUT AND LICENSES") - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - - # report issues - for name, version in src_name_ver_not_fetched: - print(f"{name}=={version}: Failed to fetch source distribution.") - - for package, envt in package_envts_not_built: - print( - f"{package.name}=={package.version}: Failed to build wheel " - f"on {envt.operating_system} for Python {envt.python_version}" + else: + print("***FETCH*** MISSING WHEELS") + package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) + print("***FETCH*** MISSING SOURCES") + src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) + + package_envts_not_built = [] + if build_wheels: + print("***BUILD*** MISSING WHEELS") + results = utils_thirdparty.build_missing_wheels( + packages_and_envts=package_envts_not_fetched, + build_remotely=build_remotely, + remote_build_log_file=remote_build_log_file, + dest_dir=thirdparty_dir, + ) + package_envts_not_built, _wheel_filenames_built = results + + print("***ADD*** ABOUT AND LICENSES") + utils_thirdparty.add_fetch_or_update_about_and_license_files( + dest_dir=thirdparty_dir, + strip_classifiers=strip_classifiers, ) + # report issues + for name, version in src_name_ver_not_fetched: + print(f"{name}=={version}: Failed to fetch source distribution.") + + for package, envt in package_envts_not_built: + print( + f"{package.name}=={package.version}: Failed to build wheel " + f"on {envt.operating_system} for Python {envt.python_version}" + ) + print("***FIND PROBLEMS***") utils_thirdparty.find_problems(dest_dir=thirdparty_dir) diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 9545db5..fc331f6 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,11 +8,13 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import re import subprocess """ Utilities to manage requirements files and call pip. -NOTE: this should use ONLY the standard library and not import anything else. +NOTE: this should use ONLY the standard library and not import anything else +becasue this is used for boostrapping. """ @@ -27,28 +29,69 @@ def load_requirements(requirements_file="requirements.txt", force_pinned=True): return get_required_name_versions(req_lines, force_pinned) -def get_required_name_versions(requirement_lines, force_pinned=True): +def get_required_name_versions( + requirement_lines, + force_pinned=True, +): """ Yield required (name, version) tuples given a`requirement_lines` iterable of requirement text lines. Every requirement versions must be pinned if `force_pinned` is True. Otherwise un-pinned requirements are returned with a - None version + None version. + """ for req_line in requirement_lines: req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if "==" not in req_line and force_pinned: - raise Exception(f"Requirement version is not pinned: {req_line}") + if force_pinned: + if "==" not in req_line: + raise Exception(f"Requirement version is not pinned: {req_line}") name = req_line version = None else: - name, _, version = req_line.partition("==") - name = name.lower().strip() - version = version.lower().strip() + if req_line.startswith("-"): + print(f"Requirement skipped, is not supported: {req_line}") + + if "==" in req_line: + name, _, version = req_line.partition("==") + version = version.lower().strip() + else: + # FIXME: we do not support unpinned requirements yet! + name = strip_reqs(req_line) + version = None + + name = name.lower().strip() yield name, version +def strip_reqs(line): + """ + Return a name given a pip reuirement text ``line` striping version and + requirements. + + For example:: + + >>> s = strip_reqs("foo <=12, >=13,!=12.6") + >>> assert s == "foo" + """ + if "--" in line: + raise Exception(f"Unsupported requirement style: {line}") + + line = line.strip() + + ops = "> Date: Mon, 14 Feb 2022 15:38:26 +0100 Subject: [PATCH 19/62] Do not use pytest 7.0.0 which is buggy Reference: https://github.com/pytest-dev/pytest/issues/9608 Signed-off-by: Philippe Ombredanne --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index f192f22..5427f0e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,8 +37,7 @@ where=src [options.extras_require] testing = - # upstream - pytest >= 6 + pytest >= 6, != 7.0.0 pytest-xdist >= 2 docs= Sphinx>=3.3.1 From b15b6b79d965f452b14cc02578a45ba1c98138fe Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 17 Feb 2022 17:31:14 +0800 Subject: [PATCH 20/62] Update `configure` to work with space in installation path Signed-off-by: Chin Yeung Li --- configure.bat | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/configure.bat b/configure.bat index 4dfb201..7e80e98 100644 --- a/configure.bat +++ b/configure.bat @@ -49,11 +49,11 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -if exist ""%CFG_ROOT_DIR%\thirdparty"" ( - set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty " +if exist "%CFG_ROOT_DIR%\thirdparty" ( + set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" --find-links https://thirdparty.aboutcode.org/pypi & %INDEX_ARG% @rem ################################ @@ -67,7 +67,7 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point set CFG_DEV_MODE=0 -set "CFG_REQUIREMENTS=%REQUIREMENTS%" +set CFG_REQUIREMENTS=%REQUIREMENTS% set "NO_INDEX=--no-index" :again @@ -75,7 +75,7 @@ if not "%1" == "" ( if "%1" EQU "--help" (goto cli_help) if "%1" EQU "--clean" (goto clean) if "%1" EQU "--dev" ( - set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" + set CFG_REQUIREMENTS=%DEV_REQUIREMENTS% set CFG_DEV_MODE=1 ) if "%1" EQU "--init" ( @@ -94,8 +94,8 @@ set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" @rem # Otherwise the latest Python by default. if not defined PYTHON_EXECUTABLE ( @rem # check for a file named PYTHON_EXECUTABLE - if exist ""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" ( - set /p PYTHON_EXECUTABLE=<""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" + if exist "%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" ( + set /p PYTHON_EXECUTABLE=<"%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" ) else ( set "PYTHON_EXECUTABLE=py" ) @@ -107,12 +107,12 @@ if not defined PYTHON_EXECUTABLE ( @rem # presence is not consistent across Linux distro and sometimes pip is not @rem # included either by default. The virtualenv.pyz app cures all these issues. -if not exist ""%CFG_BIN_DIR%\python.exe"" ( +if not exist "%CFG_BIN_DIR%\python.exe" ( if not exist "%CFG_BIN_DIR%" ( - mkdir %CFG_BIN_DIR% + mkdir "%CFG_BIN_DIR%" ) - if exist ""%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz"" ( + if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ --wheel embed --pip embed --setuptools embed ^ --seeder pip ^ @@ -120,9 +120,9 @@ if not exist ""%CFG_BIN_DIR%\python.exe"" ( --no-periodic-update ^ --no-vcs-ignore ^ %CFG_QUIET% ^ - %CFG_ROOT_DIR%\%VIRTUALENV_DIR% + "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" ) else ( - if not exist ""%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz"" ( + if not exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ( curl -o "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" %VIRTUALENV_PYZ_URL% if %ERRORLEVEL% neq 0 ( @@ -136,7 +136,7 @@ if not exist ""%CFG_BIN_DIR%\python.exe"" ( --no-periodic-update ^ --no-vcs-ignore ^ %CFG_QUIET% ^ - %CFG_ROOT_DIR%\%VIRTUALENV_DIR% + "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" ) ) @@ -152,7 +152,15 @@ if %ERRORLEVEL% neq 0 ( @rem # speeds up the installation. @rem # We always have the PEP517 build dependencies installed already. -%CFG_BIN_DIR%\pip install ^ +echo "%CFG_BIN_DIR%\pip" install ^ + --upgrade ^ + --no-build-isolation ^ + %CFG_QUIET% ^ + %PIP_EXTRA_ARGS% ^ + %CFG_REQUIREMENTS% + + +"%CFG_BIN_DIR%\pip" install ^ --upgrade ^ --no-build-isolation ^ %CFG_QUIET% ^ @@ -163,7 +171,7 @@ if %ERRORLEVEL% neq 0 ( if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ) -mklink /J %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts +mklink /J "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" if %ERRORLEVEL% neq 0 ( exit /b %ERRORLEVEL% From 311b0a16119f84c989ab1246c838c96a72fe4d06 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 17 Feb 2022 17:32:40 +0800 Subject: [PATCH 21/62] Remove echo statement Signed-off-by: Chin Yeung Li --- configure.bat | 8 -------- 1 file changed, 8 deletions(-) diff --git a/configure.bat b/configure.bat index 7e80e98..6f44967 100644 --- a/configure.bat +++ b/configure.bat @@ -152,14 +152,6 @@ if %ERRORLEVEL% neq 0 ( @rem # speeds up the installation. @rem # We always have the PEP517 build dependencies installed already. -echo "%CFG_BIN_DIR%\pip" install ^ - --upgrade ^ - --no-build-isolation ^ - %CFG_QUIET% ^ - %PIP_EXTRA_ARGS% ^ - %CFG_REQUIREMENTS% - - "%CFG_BIN_DIR%\pip" install ^ --upgrade ^ --no-build-isolation ^ From 5351f0b11d4fc83ec4469108889f4bda4bba89db Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Thu, 17 Feb 2022 19:08:29 +0530 Subject: [PATCH 22/62] automate pypi release on a tag Signed-off-by: Tushar Goel --- .github/workflows/pypi-release.yml | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/pypi-release.yml diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 0000000..b668b2e --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,36 @@ +name: Release uinvers on PyPI + +on: + release: + types: [created] + +jobs: + build-n-publish: + name: Build and publish univers to PyPI + runs-on: ubuntu-20.04 + +steps: + - uses: actions/checkout@master + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + - name: Install pypa/build + run: >- + python -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + . + - name: Publish distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file From 96b6405269f6b06911bf13194f49ba484f641be3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:49:48 +0100 Subject: [PATCH 23/62] Refine GH Action definition * Do not use explicit references to Python version and project name in descriptions * Use Python 3.8 as a base * Use only plain ASCII Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b668b2e..33eebd1 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,4 +1,4 @@ -name: Release uinvers on PyPI +name: Release library as a PyPI wheel and sdist on tag on: release: @@ -6,15 +6,15 @@ on: jobs: build-n-publish: - name: Build and publish univers to PyPI + name: Build and publish library to PyPI runs-on: ubuntu-20.04 steps: - uses: actions/checkout@master - - name: Set up Python 3.6 + - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: 3.8 - name: Install pypa/build run: >- python -m @@ -29,8 +29,9 @@ steps: --wheel --outdir dist/ . - - name: Publish distribution 📦 to PyPI + - name: Publish distribution to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@master with: - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + password: ${{ secrets.PYPI_API_TOKEN }} + \ No newline at end of file From 635df78840575777c2509525651c0d18e9771e38 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:54:51 +0100 Subject: [PATCH 24/62] Format GH action yaml The indentations were not correct Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 42 ++++++++++++------------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 33eebd1..cb32987 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -8,30 +8,20 @@ jobs: build-n-publish: name: Build and publish library to PyPI runs-on: ubuntu-20.04 - -steps: - - uses: actions/checkout@master - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install pypa/build - run: >- - python -m - pip install - build - --user - - name: Build a binary wheel and a source tarball - run: >- - python -m - build - --sdist - --wheel - --outdir dist/ + steps: + - uses: actions/checkout@master + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Install pypa/build + run: python -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python -m build --sdist --wheel --outdir dist/ . - - name: Publish distribution to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.PYPI_API_TOKEN }} - \ No newline at end of file + - name: Publish distribution to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} + From 6bbedddfeef1e10383eef87131a162b7fb43df46 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 18 Feb 2022 13:56:07 +0100 Subject: [PATCH 25/62] Use verbose name for job Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index cb32987..188497e 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -5,7 +5,7 @@ on: types: [created] jobs: - build-n-publish: + build-and-publish-to-pypi: name: Build and publish library to PyPI runs-on: ubuntu-20.04 steps: From 840ccefe0d5bbb20e9a9f7314b707bdefcd1e163 Mon Sep 17 00:00:00 2001 From: keshav-space Date: Tue, 22 Feb 2022 15:23:29 +0530 Subject: [PATCH 26/62] Add black codestyle test for skeleton - see https://github.com/nexB/skeleton/issues/54 Signed-off-by: keshav-space --- etc/scripts/fix_thirdparty.py | 6 +++--- etc/scripts/utils_requirements.py | 2 +- etc/scripts/utils_thirdparty.py | 4 +++- setup.cfg | 1 + tests/test_skeleton_codestyle.py | 36 +++++++++++++++++++++++++++++++ 5 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 tests/test_skeleton_codestyle.py diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py index 9d401cd..d664c9c 100644 --- a/etc/scripts/fix_thirdparty.py +++ b/etc/scripts/fix_thirdparty.py @@ -78,7 +78,7 @@ def fix_thirdparty_dir( package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) print("***FETCH*** MISSING SOURCES") src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - + package_envts_not_built = [] if build_wheels: print("***BUILD*** MISSING WHEELS") @@ -89,7 +89,7 @@ def fix_thirdparty_dir( dest_dir=thirdparty_dir, ) package_envts_not_built, _wheel_filenames_built = results - + print("***ADD*** ABOUT AND LICENSES") utils_thirdparty.add_fetch_or_update_about_and_license_files( dest_dir=thirdparty_dir, @@ -99,7 +99,7 @@ def fix_thirdparty_dir( # report issues for name, version in src_name_ver_not_fetched: print(f"{name}=={version}: Failed to fetch source distribution.") - + for package, envt in package_envts_not_built: print( f"{package.name}=={package.version}: Failed to build wheel " diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index fc331f6..7753ea0 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -86,7 +86,7 @@ def has_ops(l): return any(op in l for op in ops) if not has_ops: - return line + return line splitter = re.compile(r"[>= 6, != 7.0.0 pytest-xdist >= 2 + black docs= Sphinx>=3.3.1 sphinx-rtd-theme>=0.5.0 diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py new file mode 100644 index 0000000..2eb6e55 --- /dev/null +++ b/tests/test_skeleton_codestyle.py @@ -0,0 +1,36 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import subprocess +import unittest +import configparser + + +class BaseTests(unittest.TestCase): + def test_skeleton_codestyle(self): + """ + This test shouldn't run in proliferated repositories. + """ + setup_cfg = configparser.ConfigParser() + setup_cfg.read("setup.cfg") + if setup_cfg["metadata"]["name"] != "skeleton": + return + + args = "venv/bin/black --check -l 100 setup.py etc tests" + try: + subprocess.check_output(args.split()) + except subprocess.CalledProcessError as e: + print("===========================================================") + print(e.output) + print("===========================================================") + raise Exception( + "Black style check failed; please format the code using:\n" + " python -m black -l 100 setup.py etc tests", + e.output, + ) from e From 35af6431aecf7eb19f73ea14b74955cc121464b9 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 23 Feb 2022 15:42:00 +0800 Subject: [PATCH 27/62] Update configure.bat Signed-off-by: Chin Yeung Li --- configure.bat | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/configure.bat b/configure.bat index 6f44967..ed06161 100644 --- a/configure.bat +++ b/configure.bat @@ -52,9 +52,7 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) - -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" --find-links https://thirdparty.aboutcode.org/pypi & %INDEX_ARG% -@rem ################################ +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% @rem ################################ @@ -67,7 +65,7 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point set CFG_DEV_MODE=0 -set CFG_REQUIREMENTS=%REQUIREMENTS% +set "CFG_REQUIREMENTS=%REQUIREMENTS%" set "NO_INDEX=--no-index" :again @@ -75,7 +73,7 @@ if not "%1" == "" ( if "%1" EQU "--help" (goto cli_help) if "%1" EQU "--clean" (goto clean) if "%1" EQU "--dev" ( - set CFG_REQUIREMENTS=%DEV_REQUIREMENTS% + set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" set CFG_DEV_MODE=1 ) if "%1" EQU "--init" ( @@ -204,4 +202,4 @@ for %%F in (%CLEANABLE%) do ( rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 ) -exit /b 0 +exit /b 0 \ No newline at end of file From 9558c0c0a8855d085671bca7d01dc90385c84e95 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 23 Feb 2022 22:06:54 +0530 Subject: [PATCH 28/62] Deprecate windows-2016 images for azure CI Modifies the azure CI for `vs2017-win2016` to `windows-2022` as the former will be deprecated very soon. --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7cd3025..5df8a18 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,8 +41,8 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: win2016_cpython - image_name: vs2017-win2016 + job_name: win2022_cpython + image_name: windows-2022 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From e19a520a80599fd60bc0fcc3cebd2a789646fda7 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 1 Mar 2022 12:45:59 -0800 Subject: [PATCH 29/62] Remove macos 10.14 job from azure-pipelines.yml Signed-off-by: Jono Yang --- azure-pipelines.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7cd3025..bceb4ba 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,14 +23,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos1014_cpython - image_name: macos-10.14 - python_versions: ['3.6', '3.7', '3.8', '3.9'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos1015_cpython From 47da14bbae133137bba8d19b6d8c572cb28f3f7a Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 1 Mar 2022 13:10:59 -0800 Subject: [PATCH 30/62] Do not use Python 3.6 on Windows 2022 jobs * Python 3.6 is not available on Windows 2022 images Signed-off-by: Jono Yang --- azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cf057b8..f3fd2c3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,16 +33,16 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: win2022_cpython - image_name: windows-2022 + job_name: win2019_cpython + image_name: windows-2019 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ['3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From ad17a42320a8c9e8fd949f10c7b6d0019d035c24 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 23 Feb 2022 22:06:54 +0530 Subject: [PATCH 31/62] Deprecate windows-2016 images for azure CI * Modifies the azure CI for `vs2017-win2016` to `windows-2022` as the former will be deprecated very soon. Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7cd3025..5df8a18 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,8 +41,8 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: win2016_cpython - image_name: vs2017-win2016 + job_name: win2022_cpython + image_name: windows-2022 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From cad31644f17aa0af65b5c26c01cdb282f9db0951 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 1 Mar 2022 12:45:59 -0800 Subject: [PATCH 32/62] Remove macos 10.14 job from azure-pipelines.yml Signed-off-by: Jono Yang --- azure-pipelines.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5df8a18..cf057b8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,14 +23,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos1014_cpython - image_name: macos-10.14 - python_versions: ['3.6', '3.7', '3.8', '3.9'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos1015_cpython From d659e09be2ff73599d951350d17d4e5c7b72c80c Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 1 Mar 2022 13:10:59 -0800 Subject: [PATCH 33/62] Do not use Python 3.6 on Windows 2022 jobs * Python 3.6 is not available on Windows 2022 images Signed-off-by: Jono Yang --- azure-pipelines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cf057b8..f3fd2c3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,16 +33,16 @@ jobs: - template: etc/ci/azure-win.yml parameters: - job_name: win2022_cpython - image_name: windows-2022 + job_name: win2019_cpython + image_name: windows-2019 python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ['3.7', '3.8', '3.9', '3.10'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From c5251f4762d43cb88ff02636759ff7df991ead05 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Mar 2022 08:22:31 +0100 Subject: [PATCH 34/62] Run tests on macOS 11 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f3fd2c3..089abe9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -31,6 +31,14 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos11_cpython + image_name: macos-11 + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-win.yml parameters: job_name: win2019_cpython From a118fe76e3b20a778803d4630222dbf7801c30ae Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Mar 2022 08:53:32 +0100 Subject: [PATCH 35/62] Align configuration scripts on POSIX and Windows Doing so helps with keeping them in sync Signed-off-by: Philippe Ombredanne --- configure | 156 +++++++++++++++++++++++++++----------------------- configure.bat | 17 ++++-- 2 files changed, 97 insertions(+), 76 deletions(-) diff --git a/configure b/configure index fdfdc85..c1d36aa 100755 --- a/configure +++ b/configure @@ -16,6 +16,8 @@ set -e # Source this script for initial configuration # Use configure --help for details # +# NOTE: please keep in sync with Windows script configure.bat +# # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default ################################ @@ -32,10 +34,8 @@ DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constrai # where we create a virtualenv VIRTUALENV_DIR=venv -# Cleanable files and directories with the --clean option -CLEANABLE=" - build - venv" +# Cleanable files and directories to delete with the --clean option +CLEANABLE="build venv" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -50,11 +50,14 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + +################################ +# Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty " fi -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi" +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" ################################ @@ -65,56 +68,50 @@ fi ################################ -# find a proper Python to run -# Use environment variables or a file if available. -# Otherwise the latest Python by default. -if [[ "$PYTHON_EXECUTABLE" == "" ]]; then - # check for a file named PYTHON_EXECUTABLE - if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then - PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") - else - PYTHON_EXECUTABLE=python3 - fi -fi - +# Main command line entry point +main() { + CFG_REQUIREMENTS=$REQUIREMENTS + NO_INDEX="--no-index" + + # We are using getopts to parse option arguments that start with "-" + while getopts :-: optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help ) cli_help;; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + init ) NO_INDEX="";; + esac;; + esac + done -################################ -cli_help() { - echo An initial configuration script - echo " usage: ./configure [options]" - echo - echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. - echo - echo The options are: - echo " --clean: clean built and installed files and exit." - echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." - echo " --help: display this help message and exit." - echo - echo By default, the python interpreter version found in the path is used. - echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to - echo configure another Python executable interpreter to use. If this is not - echo set, a file named PYTHON_EXECUTABLE containing a single line with the - echo path of the Python executable to use will be checked last. - set +e - exit + PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" + + find_python + create_virtualenv "$VIRTUALENV_DIR" + install_packages "$CFG_REQUIREMENTS" + . "$CFG_BIN_DIR/activate" } -clean() { - # Remove cleanable file and directories and files from the root dir. - echo "* Cleaning ..." - for cln in $CLEANABLE; - do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; - done - set +e - exit +################################ +# Find a proper Python to run +# Use environment variables or a file if available. +# Otherwise the latest Python by default. +find_python() { + if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi + fi } +################################ create_virtualenv() { # create a virtualenv for Python # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -145,6 +142,7 @@ create_virtualenv() { } +################################ install_packages() { # install requirements in virtualenv # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -162,28 +160,44 @@ install_packages() { ################################ -# Main command line entry point -CFG_DEV_MODE=0 -CFG_REQUIREMENTS=$REQUIREMENTS -NO_INDEX="--no-index" - -# We are using getopts to parse option arguments that start with "-" -while getopts :-: optchar; do - case "${optchar}" in - -) - case "${OPTARG}" in - help ) cli_help;; - clean ) clean;; - dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; - init ) NO_INDEX="";; - esac;; - esac -done - -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" - -create_virtualenv "$VIRTUALENV_DIR" -install_packages "$CFG_REQUIREMENTS" -. "$CFG_BIN_DIR/activate" +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo Use the --init option if starting a new project and the project + echo dependencies are not available on thirdparty.aboutcode.org/pypi/ + echo and requirements.txt and/or requirements-dev.txt has not been generated. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --init: pull dependencies from PyPI. Used when first setting up a project." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit +} + + +################################ +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + set +e + exit +} + + + +main set +e diff --git a/configure.bat b/configure.bat index ed06161..961e0d9 100644 --- a/configure.bat +++ b/configure.bat @@ -14,6 +14,8 @@ @rem # Source this script for initial configuration @rem # Use configure --help for details +@rem # NOTE: please keep in sync with POSIX script configure + @rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz @rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default @rem ################################ @@ -49,10 +51,11 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling +@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org if exist "%CFG_ROOT_DIR%\thirdparty" ( set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @rem ################################ @@ -64,7 +67,6 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point -set CFG_DEV_MODE=0 set "CFG_REQUIREMENTS=%REQUIREMENTS%" set "NO_INDEX=--no-index" @@ -74,7 +76,6 @@ if not "%1" == "" ( if "%1" EQU "--clean" (goto clean) if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" - set CFG_DEV_MODE=1 ) if "%1" EQU "--init" ( set "NO_INDEX= " @@ -87,7 +88,7 @@ set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" @rem ################################ -@rem # find a proper Python to run +@rem # Find a proper Python to run @rem # Use environment variables or a file if available. @rem # Otherwise the latest Python by default. if not defined PYTHON_EXECUTABLE ( @@ -99,6 +100,8 @@ if not defined PYTHON_EXECUTABLE ( ) ) + +@rem ################################ :create_virtualenv @rem # create a virtualenv for Python @rem # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -143,6 +146,7 @@ if %ERRORLEVEL% neq 0 ( ) +@rem ################################ :install_packages @rem # install requirements in virtualenv @rem # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -157,6 +161,9 @@ if %ERRORLEVEL% neq 0 ( %PIP_EXTRA_ARGS% ^ %CFG_REQUIREMENTS% + +@rem ################################ +:create_bin_junction @rem # Create junction to bin to have the same directory between linux and windows if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" @@ -171,7 +178,6 @@ exit /b 0 @rem ################################ - :cli_help echo An initial configuration script echo " usage: configure [options]" @@ -195,6 +201,7 @@ exit /b 0 exit /b 0 +@rem ################################ :clean @rem # Remove cleanable file and directories and files from the root dir. echo "* Cleaning ..." From e810da356b99cb7241c90c9a79b20232ddebbb50 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 4 Mar 2022 09:00:46 +0100 Subject: [PATCH 36/62] Update README Signed-off-by: Philippe Ombredanne --- README.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 4173689..74e58fa 100644 --- a/README.rst +++ b/README.rst @@ -1,19 +1,27 @@ A Simple Python Project Skeleton ================================ -This repo attempts to standardize our python repositories using modern python -packaging and configuration techniques. Using this `blog post`_ as inspiration, this -repository will serve as the base for all new python projects and will be adopted to all -our existing ones as well. +This repo attempts to standardize the structure of the Python-based project's +repositories using modern Python packaging and configuration techniques. +Using this `blog post`_ as inspiration, this repository serves as the base for +all new Python projects and is mergeable in existing repositories as well. .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ + Usage ===== Usage instructions can be found in ``docs/skeleton-usage.rst``. + Release Notes ============= +- 2022-03-04: + - Synchronize configure and configure.bat scripts for sanity + - Update CI operating system support with latest Azure OS images + - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies + There are now fewer scripts. See etc/scripts/README.rst for details + - 2021-09-03: - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` - ``configure`` can now accept multiple options at once From 243f7cb96ec9ab03080ffb8e69197b80894ef565 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 5 Mar 2022 07:07:18 +0100 Subject: [PATCH 37/62] Refactor and streamline thirdparty utilities There is now a single primary script "fetch_thirdparty.py" that handles everything in a simplified way. There is no utility to build wheels anymore: these MUST be available before hand on PyPI or built manually and uploaded in our self-hosted PyPI repository. Signed-off-by: Philippe Ombredanne --- etc/scripts/README.rst | 45 +- etc/scripts/bootstrap.py | 244 ---- etc/scripts/build_wheels.py | 121 -- etc/scripts/check_thirdparty.py | 30 +- etc/scripts/fetch_built_wheels.py | 63 - etc/scripts/fetch_requirements.py | 159 --- etc/scripts/fetch_thirdparty.py | 306 +++++ etc/scripts/fix_thirdparty.py | 114 -- etc/scripts/gen_pypi_simple.py | 246 +++- etc/scripts/publish_files.py | 208 --- etc/scripts/utils_requirements.py | 147 +- etc/scripts/utils_thirdparty.py | 2079 +++++++++-------------------- 12 files changed, 1233 insertions(+), 2529 deletions(-) delete mode 100644 etc/scripts/bootstrap.py delete mode 100644 etc/scripts/build_wheels.py delete mode 100644 etc/scripts/fetch_built_wheels.py delete mode 100644 etc/scripts/fetch_requirements.py create mode 100644 etc/scripts/fetch_thirdparty.py delete mode 100644 etc/scripts/fix_thirdparty.py delete mode 100644 etc/scripts/publish_files.py diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index d8b00f9..edf82e4 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -15,10 +15,10 @@ Pre-requisites * To generate or update pip requirement files, you need to start with a clean virtualenv as instructed below (This is to avoid injecting requirements - specific to the tools here in the main requirements). + specific to the tools used here in the main requirements). * For other usages, the tools here can run either in their own isolated - virtualenv best or in the the main configured development virtualenv. + virtualenv or in the the main configured development virtualenv. These requireements need to be installed:: pip install --requirement etc/release/requirements.txt @@ -82,45 +82,14 @@ Populate a thirdparty directory with wheels, sources, .ABOUT and license files Scripts ~~~~~~~ -* **fetch_requirements.py** will fetch package wheels, their ABOUT, LICENSE and - NOTICE files to populate a local a thirdparty directory strictly from our - remote repo and using only pinned packages listed in one or more pip - requirements file(s). Fetch only requirements for specific python versions and - operating systems. Optionally fetch the corresponding source distributions. - -* **publish_files.py** will upload/sync a thirdparty directory of files to our - remote repo. Requires a GitHub personal access token. - -* **build_wheels.py** will build a package binary wheel for multiple OS and - python versions. Optionally wheels that contain native code are built - remotely. Dependent wheels are optionally included. Requires Azure credentials - and tokens if building wheels remotely on multiple operatin systems. - -* **fix_thirdparty.py** will fix a thirdparty directory with a best effort to - add missing wheels, sources archives, create or fetch or fix .ABOUT, .NOTICE - and .LICENSE files. Requires Azure credentials and tokens if requesting the - build of missing wheels remotely on multiple operatin systems. +* **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs + and their ABOUT, LICENSE and NOTICE files to populate a local directory from + a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI) + using pip requirements file(s), specifiers or pre-existing packages files. + Fetch wheels for specific python version and operating system combinations. * **check_thirdparty.py** will check a thirdparty directory for errors. -* **bootstrap.py** will bootstrap a thirdparty directory from a requirements - file(s) to add or build missing wheels, sources archives and create .ABOUT, - .NOTICE and .LICENSE files. Requires Azure credentials and tokens if - requesting the build of missing wheels remotely on multiple operatin systems. - - - -Usage -~~~~~ - -See each command line --help option for details. - -* (TODO) **add_package.py** will add or update a Python package including wheels, - sources and ABOUT files and this for multiple Python version and OSes(for use - with upload_packages.py afterwards) You will need an Azure personal access - token for buidling binaries and an optional DejaCode API key to post and fetch - new package versions there. TODO: explain how we use romp - Upgrade virtualenv app ---------------------- diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py deleted file mode 100644 index 31f2f55..0000000 --- a/etc/scripts/bootstrap.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import itertools - -import click - -import utils_thirdparty -from utils_thirdparty import Environment -from utils_thirdparty import PypiPackage - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file(s) to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built and " - "sources, ABOUT and LICENSE files fetched.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version(s) to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS(ses) to use for this build: one of linux, mac or windows.", -) -@click.option( - "-l", - "--latest-version", - is_flag=True, - help="Get the latest version of all packages, ignoring version specifiers.", -) -@click.option( - "--sync-dejacode", - is_flag=True, - help="Synchronize packages with DejaCode.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.help_option("-h", "--help") -def bootstrap( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_deps, - latest_version, - sync_dejacode, - build_remotely=False, -): - """ - Boostrap a thirdparty Python packages directory from pip requirements. - - Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for - the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible - with all the provided ``--python-version`` PYVER(s) and ```--operating_system`` - OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and - .LICENSE files. - - Optionally ignore version specifiers and use the ``--latest-version`` - of everything. - - Sources and wheels are fetched with attempts first from PyPI, then our remote repository. - If missing wheels are built as needed. - """ - # rename variables for clarity since these are lists - requirements_files = requirements_file - python_versions = python_version - operating_systems = operating_system - - # create the environments we need - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - - # collect all packages to process from requirements files - # this will fail with an exception if there are packages we cannot find - - required_name_versions = set() - - for req_file in requirements_files: - nvs = utils_thirdparty.load_requirements(requirements_file=req_file, force_pinned=False) - required_name_versions.update(nvs) - if latest_version: - required_name_versions = set((name, None) for name, _ver in required_name_versions) - - print( - f"PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES" - ) - - # fetch all available wheels, keep track of missing - # start with local, then remote, then PyPI - - print("==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS") - # list of all the wheel filenames either pre-existing, fetched or built - # updated as we progress - available_wheel_filenames = [] - - local_packages_by_namever = { - (p.name, p.version): p - for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir) - } - - # list of (name, version, environment) not local and to fetch - name_version_envt_to_fetch = [] - - # start with a local check - for (name, version), envt in itertools.product(required_name_versions, environments): - local_pack = local_packages_by_namever.get( - ( - name, - version, - ) - ) - if local_pack: - supported_wheels = list(local_pack.get_supported_wheels(environment=envt)) - if supported_wheels: - available_wheel_filenames.extend(w.filename for w in supported_wheels) - print( - f"====> No fetch or build needed. " - f"Local wheel already available for {name}=={version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - continue - - name_version_envt_to_fetch.append( - ( - name, - version, - envt, - ) - ) - - print(f"==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS") - - # list of (name, version, environment) not fetch and to build - name_version_envt_to_build = [] - - # then check if the wheel can be fetched without building from remote and Pypi - for name, version, envt in name_version_envt_to_fetch: - - fetched_fwn = utils_thirdparty.fetch_package_wheel( - name=name, - version=version, - environment=envt, - dest_dir=thirdparty_dir, - ) - - if fetched_fwn: - available_wheel_filenames.append(fetched_fwn) - else: - name_version_envt_to_build.append( - ( - name, - version, - envt, - ) - ) - - # At this stage we have all the wheels we could obtain without building - for name, version, envt in name_version_envt_to_build: - print( - f"====> Need to build wheels for {name}=={version} on os: " - f"{envt.operating_system} for Python: {envt.python_version}" - ) - - packages_and_envts_to_build = [ - (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build - ] - - print(f"==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS") - - package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( - packages_and_envts=packages_and_envts_to_build, - build_remotely=build_remotely, - with_deps=with_deps, - dest_dir=thirdparty_dir, - ) - if wheel_filenames_built: - available_wheel_filenames.extend(available_wheel_filenames) - - for pack, envt in package_envts_not_built: - print( - f"====> FAILED to build any wheel for {pack.name}=={pack.version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - - print(f"==> FETCHING SOURCE DISTRIBUTIONS") - # fetch all sources, keep track of missing - # This is a list of (name, version) - utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - print(f"==> FETCHING ABOUT AND LICENSE FILES") - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - - ############################################################################ - if sync_dejacode: - print(f"==> SYNC WITH DEJACODE") - # try to fetch from DejaCode any missing ABOUT - # create all missing DejaCode packages - pass - - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - bootstrap() diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py deleted file mode 100644 index 8a28176..0000000 --- a/etc/scripts/build_wheels.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-n", - "--name", - type=str, - metavar="PACKAGE_NAME", - required=True, - help="Python package name to add or build.", -) -@click.option( - "-v", - "--version", - type=str, - default=None, - metavar="VERSION", - help="Python package version to add or build.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def build_wheels( - name, - version, - thirdparty_dir, - python_version, - operating_system, - with_deps, - build_remotely, - remote_build_log_file, - verbose, -): - """ - Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and - optional VERSION. Build wheels compatible with all the `--python-version` - PYVER(s) and `--operating_system` OS(s). - - Build native wheels remotely if needed when `--build-remotely` and include - all dependencies with `--with-deps`. - """ - utils_thirdparty.add_or_upgrade_built_wheels( - name=name, - version=version, - python_versions=python_version, - operating_systems=operating_system, - dest_dir=thirdparty_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - -if __name__ == "__main__": - build_wheels() diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 4fea16c..0f04b34 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,17 +16,39 @@ @click.command() @click.option( "-d", - "--thirdparty-dir", + "--dest_dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Check missing wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Check missing source sdists tarballs.", +) @click.help_option("-h", "--help") -def check_thirdparty_dir(thirdparty_dir): +def check_thirdparty_dir( + dest_dir, + wheels, + sdists, +): """ - Check a thirdparty directory for problems. + Check a thirdparty directory for problems and print these on screen. """ - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) if __name__ == "__main__": diff --git a/etc/scripts/fetch_built_wheels.py b/etc/scripts/fetch_built_wheels.py deleted file mode 100644 index a78861e..0000000 --- a/etc/scripts/fetch_built_wheels.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "--remote-build-log-file", - type=click.Path(readable=True), - metavar="LOG-FILE", - help="Path to a remote builds log file.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory to save built wheels.", -) -@click.option( - "--no-wait", - is_flag=True, - default=False, - help="Do not wait for build completion.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def fetch_remote_wheels( - remote_build_log_file, - thirdparty_dir, - no_wait, - verbose, -): - """ - Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines - build log file. - """ - utils_thirdparty.fetch_remotely_built_wheels( - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - no_wait=no_wait, - verbose=verbose, - ) - - -if __name__ == "__main__": - fetch_remote_wheels() diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py deleted file mode 100644 index 9da9ce9..0000000 --- a/etc/scripts/fetch_requirements.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import itertools - -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="INT", - multiple=True, - default=["36"], - show_default=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - multiple=True, - default=["linux"], - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "-s", - "--with-sources", - is_flag=True, - help="Fetch the corresponding source distributions.", -) -@click.option( - "-a", - "--with-about", - is_flag=True, - help="Fetch the corresponding ABOUT and LICENSE files.", -) -@click.option( - "--allow-unpinned", - is_flag=True, - help="Allow requirements without pinned versions.", -) -@click.option( - "-s", - "--only-sources", - is_flag=True, - help="Fetch only the corresponding source distributions.", -) -@click.option( - "-u", - "--remote-links-url", - type=str, - metavar="URL", - default=utils_thirdparty.REMOTE_LINKS_URL, - show_default=True, - help="URL to a PyPI-like links web site. " "Or local path to a directory with wheels.", -) -@click.help_option("-h", "--help") -def fetch_requirements( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_sources, - with_about, - allow_unpinned, - only_sources, - remote_links_url=utils_thirdparty.REMOTE_LINKS_URL, -): - """ - Fetch and save to THIRDPARTY_DIR all the required wheels for pinned - dependencies found in the `--requirement` FILE requirements file(s). Only - fetch wheels compatible with the provided `--python-version` and - `--operating-system`. - Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together - with a virtualenv.pyz app. - - Use exclusively wheel not from PyPI but rather found in the PyPI-like link - repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` - as a local directory path to a wheels directory if this is not a a URL. - """ - - # fetch wheels - python_versions = python_version - operating_systems = operating_system - requirements_files = requirements_file - - if not only_sources: - envs = itertools.product(python_versions, operating_systems) - envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs) - - for env, reqf in itertools.product(envs, requirements_files): - - for package, error in utils_thirdparty.fetch_wheels( - environment=env, - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch wheel:", package, ":", error) - - # optionally fetch sources - if with_sources or only_sources: - - for reqf in requirements_files: - for package, error in utils_thirdparty.fetch_sources( - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch source:", package, ":", error) - - if with_about: - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - utils_thirdparty.find_problems( - dest_dir=thirdparty_dir, - report_missing_sources=with_sources or only_sources, - report_missing_wheels=not only_sources, - ) - - -if __name__ == "__main__": - fetch_requirements() diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py new file mode 100644 index 0000000..22147b2 --- /dev/null +++ b/etc/scripts/fetch_thirdparty.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import itertools +import os +import sys + +import click + +import utils_thirdparty +import utils_requirements + +TRACE = True + + +@click.command() +@click.option( + "-r", + "--requirements", + "requirements_files", + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar="REQUIREMENT-FILE", + multiple=True, + required=False, + help="Path to pip requirements file(s) listing thirdparty packages.", +) +@click.option( + "--spec", + "--specifier", + "specifiers", + type=str, + metavar="SPECIFIER", + multiple=True, + required=False, + help="Thirdparty package name==version specification(s) as in django==1.2.3. " + "With --latest-version a plain package name is also acceptable.", +) +@click.option( + "-l", + "--latest-version", + is_flag=True, + help="Get the latest version of all packages, ignoring any specified versions.", +) +@click.option( + "-d", + "--dest", + "dest_dir", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar="DIR", + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help="Path to the detsination directory where to save downloaded wheels, " + "sources, ABOUT and LICENSE files..", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Download wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Download source sdists tarballs.", +) +@click.option( + "-p", + "--python-version", + "python_versions", + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar="PYVER", + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help="Python version(s) to use for wheels.", +) +@click.option( + "-o", + "--operating-system", + "operating_systems", + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar="OS", + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help="OS(ses) to use for wheels: one of linux, mac or windows.", +) +@click.option( + "--index-url", + "index_urls", + type=str, + metavar="INDEX", + default=utils_thirdparty.PYPI_INDEXES, + show_default=True, + multiple=True, + help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", +) +@click.help_option("-h", "--help") +def fetch_thirdparty( + requirements_files, + specifiers, + latest_version, + dest_dir, + python_versions, + operating_systems, + wheels, + sdists, + index_urls, +): + """ + Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions, + and their ABOUT metadata, license and notices files. + + Download the PyPI packages listed in the combination of: + - the pip requirements --requirements REQUIREMENT-FILE(s), + - the pip name==version --specifier SPECIFIER(s) + - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. + + Download wheels with the --wheels option for the ``--python-version`` PYVER(s) + and ``--operating_system`` OS(s) combinations defaulting to all supported combinations. + + Download sdists tarballs with the --sdists option. + + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched. + + Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs. + """ + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + existing_packages_by_nv = { + (package.name, package.version): package + for package in utils_thirdparty.get_local_packages(directory=dest_dir) + } + + required_name_versions = set(existing_packages_by_nv.keys()) + + for req_file in requirements_files: + nvs = utils_requirements.load_requirements( + requirements_file=req_file, + with_unpinned=latest_version, + ) + required_name_versions.update(nvs) + + for specifier in specifiers: + nv = utils_requirements.get_name_version( + requirement=specifier, + with_unpinned=latest_version, + ) + required_name_versions.add(nv) + + if not required_name_versions: + print("Error: no requirements requested.") + sys.exit(1) + + if not os.listdir(dest_dir) and not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + + if latest_version: + latest_name_versions = set() + names = set(name for name, _version in sorted(required_name_versions)) + for name in sorted(names): + latests = utils_thirdparty.PypiPackage.sorted( + utils_thirdparty.get_package_versions( + name=name, version=None, index_urls=index_urls + ) + ) + if not latests: + print(f"No distribution found for: {name}") + continue + latest = latests[-1] + latest_name_versions.add((latest.name, latest.version)) + required_name_versions = latest_name_versions + + if TRACE: + print("required_name_versions:", required_name_versions) + + if wheels: + # create the environments matrix we need for wheels + evts = itertools.product(python_versions, operating_systems) + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + + wheels_not_found = {} + sdists_not_found = {} + # iterate over requirements, one at a time + for name, version in sorted(required_name_versions): + nv = name, version + existing_package = existing_packages_by_nv.get(nv) + if wheels: + for environment in environments: + if existing_package: + existing_wheels = list( + existing_package.get_supported_wheels(environment=environment) + ) + else: + existing_wheels = None + + if existing_wheels: + if TRACE: + print( + f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" + ) + if all(w.is_pure() for w in existing_wheels): + break + else: + continue + + if TRACE: + print(f"Fetching wheel for: {name}=={version} on: {environment}") + + try: + ( + fetched_wheel_filenames, + existing_wheel_filenames, + ) = utils_thirdparty.download_wheel( + name=name, + version=version, + environment=environment, + dest_dir=dest_dir, + index_urls=index_urls, + ) + if TRACE: + if existing_wheel_filenames: + print( + f" ====> Wheels already available: {name}=={version} on: {environment}" + ) + for whl in existing_wheel_filenames: + print(f" {whl}") + if fetched_wheel_filenames: + print(f" ====> Wheels fetched: {name}=={version} on: {environment}") + for whl in fetched_wheel_filenames: + print(f" {whl}") + + fwfns = fetched_wheel_filenames + existing_wheel_filenames + + if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns): + break + + except utils_thirdparty.DistributionNotFound as e: + wheels_not_found[f"{name}=={version}"] = str(e) + + if sdists: + if existing_package and existing_package.sdist: + if TRACE: + print( + f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}" + ) + continue + + if TRACE: + print(f" Fetching sdist for: {name}=={version}") + + try: + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + index_urls=index_urls, + ) + + if TRACE: + if not fetched: + print( + f" ====> Sdist already available: {name}=={version} on: {environment}" + ) + else: + print( + f" ====> Sdist fetched: {fetched} for {name}=={version} on: {environment}" + ) + + except utils_thirdparty.DistributionNotFound as e: + sdists_not_found[f"{name}=={version}"] = str(e) + + if wheels and wheels_not_found: + print(f"==> MISSING WHEELS") + for wh in wheels_not_found: + print(f" {wh}") + + if sdists and sdists_not_found: + print(f"==> MISSING SDISTS") + for sd in sdists_not_found: + print(f" {sd}") + + print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir) + utils_thirdparty.clean_about_files(dest_dir=dest_dir) + + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + fetch_thirdparty() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py deleted file mode 100644 index d664c9c..0000000 --- a/etc/scripts/fix_thirdparty.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - required=True, - help="Path to the thirdparty directory to fix.", -) -@click.option( - "--build-wheels", - is_flag=True, - help="Build all missing wheels .", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--strip-classifiers", - is_flag=True, - help="Remove danglingf classifiers", -) -@click.help_option("-h", "--help") -def fix_thirdparty_dir( - thirdparty_dir, - build_wheels, - build_remotely, - remote_build_log_file, - strip_classifiers, -): - """ - Fix a thirdparty directory of dependent package wheels and sdist. - - Multiple fixes are applied: - - fetch or build missing binary wheels - - fetch missing source distributions - - derive, fetch or add missing ABOUT files - - fetch missing .LICENSE and .NOTICE files - - remove outdated package versions and the ABOUT, .LICENSE and .NOTICE files - - Optionally build missing binary wheels for all supported OS and Python - version combos locally or remotely. - """ - if strip_classifiers: - print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - else: - print("***FETCH*** MISSING WHEELS") - package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) - print("***FETCH*** MISSING SOURCES") - src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - package_envts_not_built = [] - if build_wheels: - print("***BUILD*** MISSING WHEELS") - results = utils_thirdparty.build_missing_wheels( - packages_and_envts=package_envts_not_fetched, - build_remotely=build_remotely, - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - ) - package_envts_not_built, _wheel_filenames_built = results - - print("***ADD*** ABOUT AND LICENSES") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - - # report issues - for name, version in src_name_ver_not_fetched: - print(f"{name}=={version}: Failed to fetch source distribution.") - - for package, envt in package_envts_not_built: - print( - f"{package.name}=={package.version}: Failed to build wheel " - f"on {envt.operating_system} for Python {envt.python_version}" - ) - - print("***FIND PROBLEMS***") - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - fix_thirdparty_dir() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 53db9b0..8de2b96 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -5,65 +5,25 @@ # Copyright (c) 2010 David Wolever . All rights reserved. # originally from https://github.com/wolever/pip2pi +import hashlib import os import re import shutil - +from collections import defaultdict from html import escape from pathlib import Path +from typing import NamedTuple """ -name: pip compatibility tags -version: 20.3.1 -download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py -copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) -license_expression: mit -notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py - -Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Generate a PyPI simple index froma directory. """ -get_wheel_from_filename = re.compile( - r"""^(?P(?P.+?)-(?P.*?)) - ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) - \.whl)$""", - re.VERBOSE, -).match - -sdist_exts = ( - ".tar.gz", - ".tar.bz2", - ".zip", - ".tar.xz", -) -wheel_ext = ".whl" -app_ext = ".pyz" -dist_exts = sdist_exts + (wheel_ext, app_ext) class InvalidDistributionFilename(Exception): pass -def get_package_name_from_filename(filename, normalize=True): +def get_package_name_from_filename(filename): """ Return the package name extracted from a package ``filename``. Optionally ``normalize`` the name according to distribution name rules. @@ -132,18 +92,99 @@ def get_package_name_from_filename(filename, normalize=True): if not name: raise InvalidDistributionFilename(filename) - if normalize: - name = name.lower().replace("_", "-") + name = normalize_name(name) return name -def build_pypi_index(directory, write_index=False): +def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + +def build_per_package_index(pkg_name, packages, base_url): + """ + Return an HTML document as string representing the index for a package + """ + document = [] + header = f""" + + + + Links for {pkg_name} + + """ + document.append(header) + + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +def build_links_package_index(packages_by_package_name, base_url): + """ + Return an HTML document as string which is a links index of all packages + """ + document = [] + header = f""" + + + Links for all packages + + """ + document.append(header) + + for _name, packages in packages_by_package_name.items(): + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +class Package(NamedTuple): + name: str + index_dir: Path + archive_file: Path + checksum: str + + @classmethod + def from_file(cls, name, index_dir, archive_file): + with open(archive_file, "rb") as f: + checksum = hashlib.sha256(f.read()).hexdigest() + return cls( + name=name, + index_dir=index_dir, + archive_file=archive_file, + checksum=checksum, + ) + + def simple_index_entry(self, base_url): + return ( + f' ' + f"{self.archive_file.name}
" + ) + + +def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI simple - directory index at ``directory``/simple/ populated with the proper PyPI simple - index directory structure crafted using symlinks. + Using a ``directory`` directory of wheels and sdists, create the a PyPI + simple directory index at ``directory``/simple/ populated with the proper + PyPI simple index directory structure crafted using symlinks. WARNING: The ``directory``/simple/ directory is removed if it exists. + NOTE: in addition to the a PyPI simple index.html there is also a links.html + index file generated which is suitable to use with pip's --find-links """ directory = Path(directory) @@ -153,14 +194,15 @@ def build_pypi_index(directory, write_index=False): shutil.rmtree(str(index_dir), ignore_errors=True) index_dir.mkdir(parents=True) + packages_by_package_name = defaultdict(list) - if write_index: - simple_html_index = [ - "PyPI Simple Index", - "", - ] + # generate the main simple index.html + simple_html_index = [ + "", + "PyPI Simple Index", + '' '', + ] - package_names = set() for pkg_file in directory.iterdir(): pkg_filename = pkg_file.name @@ -172,23 +214,99 @@ def build_pypi_index(directory, write_index=False): ): continue - pkg_name = get_package_name_from_filename(pkg_filename) + pkg_name = get_package_name_from_filename( + filename=pkg_filename, + ) pkg_index_dir = index_dir / pkg_name pkg_index_dir.mkdir(parents=True, exist_ok=True) pkg_indexed_file = pkg_index_dir / pkg_filename + link_target = Path("../..") / pkg_filename pkg_indexed_file.symlink_to(link_target) - if write_index and pkg_name not in package_names: + if pkg_name not in packages_by_package_name: esc_name = escape(pkg_name) simple_html_index.append(f'{esc_name}
') - package_names.add(pkg_name) - if write_index: - simple_html_index.append("") - index_html = index_dir / "index.html" - index_html.write_text("\n".join(simple_html_index)) + packages_by_package_name[pkg_name].append( + Package.from_file( + name=pkg_name, + index_dir=pkg_index_dir, + archive_file=pkg_file, + ) + ) + + # finalize main index + simple_html_index.append("") + index_html = index_dir / "index.html" + index_html.write_text("\n".join(simple_html_index)) + + # also generate the simple index.html of each package, listing all its versions. + for pkg_name, packages in packages_by_package_name.items(): + per_package_index = build_per_package_index( + pkg_name=pkg_name, + packages=packages, + base_url=base_url, + ) + pkg_index_dir = packages[0].index_dir + ppi_html = pkg_index_dir / "index.html" + ppi_html.write_text(per_package_index) + + # also generate the a links.html page with all packages. + package_links = build_links_package_index( + packages_by_package_name=packages_by_package_name, + base_url=base_url, + ) + links_html = index_dir / "links.html" + links_html.write_text(package_links) + + +""" +name: pip-wheel +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, +).match + +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) if __name__ == "__main__": import sys diff --git a/etc/scripts/publish_files.py b/etc/scripts/publish_files.py deleted file mode 100644 index 8669363..0000000 --- a/etc/scripts/publish_files.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import hashlib -import os -import sys - -from pathlib import Path - -import click -import requests -import utils_thirdparty - -from github_release_retry import github_release_retry as grr - -""" -Create GitHub releases and upload files there. -""" - - -def get_files(location): - """ - Return an iterable of (filename, Path, md5) tuples for files in the `location` - directory tree recursively. - """ - for top, _dirs, files in os.walk(location): - for filename in files: - pth = Path(os.path.join(top, filename)) - with open(pth, "rb") as fi: - md5 = hashlib.md5(fi.read()).hexdigest() - yield filename, pth, md5 - - -def get_etag_md5(url): - """ - Return the cleaned etag of URL `url` or None. - """ - headers = utils_thirdparty.get_remote_headers(url) - headers = {k.lower(): v for k, v in headers.items()} - etag = headers.get("etag") - if etag: - etag = etag.strip('"').lower() - return etag - - -def create_or_update_release_and_upload_directory( - user, - repo, - tag_name, - token, - directory, - retry_limit=10, - description=None, -): - """ - Create or update a GitHub release at https://github.com// for - `tag_name` tag using the optional `description` for this release. - Use the provided `token` as a GitHub token for API calls authentication. - Upload all files found in the `directory` tree to that GitHub release. - Retry API calls up to `retry_limit` time to work around instability the - GitHub API. - - Remote files that are not the same as the local files are deleted and re- - uploaded. - """ - release_homepage_url = f"https://github.com/{user}/{repo}/releases/{tag_name}" - - # scrape release page HTML for links - urls_by_filename = { - os.path.basename(l): l - for l in utils_thirdparty.get_paths_or_urls(links_url=release_homepage_url) - } - - # compute what is new, modified or unchanged - print(f"Compute which files is new, modified or unchanged in {release_homepage_url}") - - new_to_upload = [] - unchanged_to_skip = [] - modified_to_delete_and_reupload = [] - for filename, pth, md5 in get_files(directory): - url = urls_by_filename.get(filename) - if not url: - print(f"{filename} content is NEW, will upload") - new_to_upload.append(pth) - continue - - out_of_date = get_etag_md5(url) != md5 - if out_of_date: - print(f"{url} content is CHANGED based on md5 etag, will re-upload") - modified_to_delete_and_reupload.append(pth) - else: - # print(f'{url} content is IDENTICAL, skipping upload based on Etag') - unchanged_to_skip.append(pth) - print(".") - - ghapi = grr.GithubApi( - github_api_url="https://api.github.com", - user=user, - repo=repo, - token=token, - retry_limit=retry_limit, - ) - - # yank modified - print( - f"Unpublishing {len(modified_to_delete_and_reupload)} published but " - f"locally modified files in {release_homepage_url}" - ) - - release = ghapi.get_release_by_tag(tag_name) - - for pth in modified_to_delete_and_reupload: - filename = os.path.basename(pth) - asset_id = ghapi.find_asset_id_by_file_name(filename, release) - print(f" Unpublishing file: {filename}).") - response = ghapi.delete_asset(asset_id) - if response.status_code != requests.codes.no_content: # NOQA - raise Exception(f"failed asset deletion: {response}") - - # finally upload new and modified - to_upload = new_to_upload + modified_to_delete_and_reupload - print(f"Publishing with {len(to_upload)} files to {release_homepage_url}") - release = grr.Release(tag_name=tag_name, body=description) - grr.make_release(ghapi, release, to_upload) - - -TOKEN_HELP = ( - "The Github personal acess token is used to authenticate API calls. " - "Required unless you set the GITHUB_TOKEN environment variable as an alternative. " - "See for details: https://github.com/settings/tokens and " - "https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token" -) - - -@click.command() -@click.option( - "--user-repo-tag", - help="The GitHub qualified repository user/name/tag in which " - "to create the release such as in nexB/thirdparty/pypi", - type=str, - required=True, -) -@click.option( - "-d", - "--directory", - help="The directory that contains files to upload to the release.", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, -) -@click.option( - "--token", - help=TOKEN_HELP, - default=os.environ.get("GITHUB_TOKEN", None), - type=str, - required=False, -) -@click.option( - "--description", - help="Text description for the release. Ignored if the release exists.", - default=None, - type=str, - required=False, -) -@click.option( - "--retry_limit", - help="Number of retries when making failing GitHub API calls. " - "Retrying helps work around transient failures of the GitHub API.", - type=int, - default=10, -) -@click.help_option("-h", "--help") -def publish_files( - user_repo_tag, - directory, - retry_limit=10, - token=None, - description=None, -): - """ - Publish all the files in DIRECTORY as assets to a GitHub release. - Either create or update/replace remote files' - """ - if not token: - click.secho("--token required option is missing.") - click.secho(TOKEN_HELP) - sys.exit(1) - - user, repo, tag_name = user_repo_tag.split("/") - - create_or_update_release_and_upload_directory( - user=user, - repo=repo, - tag_name=tag_name, - description=description, - retry_limit=retry_limit, - token=token, - directory=directory, - ) - - -if __name__ == "__main__": - publish_files() diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 7753ea0..fbc456d 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -14,95 +14,63 @@ """ Utilities to manage requirements files and call pip. NOTE: this should use ONLY the standard library and not import anything else -becasue this is used for boostrapping. +because this is used for boostrapping with no requirements installed. """ -def load_requirements(requirements_file="requirements.txt", force_pinned=True): +def load_requirements(requirements_file="requirements.txt", with_unpinned=False): """ Yield package (name, version) tuples for each requirement in a `requirement` - file. Every requirement versions must be pinned if `force_pinned` is True. - Otherwise un-pinned requirements are returned with a None version + file. Only accept requirements pinned to an exact version. """ with open(requirements_file) as reqs: req_lines = reqs.read().splitlines(False) - return get_required_name_versions(req_lines, force_pinned) + return get_required_name_versions(req_lines, with_unpinned=with_unpinned) -def get_required_name_versions( - requirement_lines, - force_pinned=True, -): +def get_required_name_versions(requirement_lines, with_unpinned=False): """ Yield required (name, version) tuples given a`requirement_lines` iterable of - requirement text lines. Every requirement versions must be pinned if - `force_pinned` is True. Otherwise un-pinned requirements are returned with a - None version. - + requirement text lines. Only accept requirements pinned to an exact version. """ + for req_line in requirement_lines: req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if force_pinned: - if "==" not in req_line: - raise Exception(f"Requirement version is not pinned: {req_line}") - name = req_line - version = None - else: - if req_line.startswith("-"): - print(f"Requirement skipped, is not supported: {req_line}") - - if "==" in req_line: - name, _, version = req_line.partition("==") - version = version.lower().strip() - else: - # FIXME: we do not support unpinned requirements yet! - name = strip_reqs(req_line) - version = None - - name = name.lower().strip() - yield name, version - - -def strip_reqs(line): - """ - Return a name given a pip reuirement text ``line` striping version and - requirements. - - For example:: - - >>> s = strip_reqs("foo <=12, >=13,!=12.6") - >>> assert s == "foo" - """ - if "--" in line: - raise Exception(f"Unsupported requirement style: {line}") - - line = line.strip() - - ops = ">>> assert get_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_name_version("foo", with_unpinned=True) == ("foo", "") + >>> assert get_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_name_version("foo>=1.2") + >>> try: + ... assert not get_name_version("foo", with_unpinned=False) + ... except Exception as e: + ... assert "Requirement version must be pinned" in str(e) """ - requires = [c for c in requires.splitlines(False) if c] - if not requires: - return [] - - requires = ["".join(r.split()) for r in requires if r and r.strip()] - return sorted(requires) + requirement = requirement and "".join(requirement.lower().split()) + assert requirement, f"specifier is required is empty:{requirement!r}" + name, operator, version = split_req(requirement) + assert name, f"Name is required: {requirement}" + is_pinned = operator == "==" + if with_unpinned: + version = "" + else: + assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + return name, version def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): @@ -139,8 +107,47 @@ def lock_dev_requirements( def get_installed_reqs(site_packages_dir): """ - Return the installed pip requirements as text found in `site_packages_dir` as a text. + Return the installed pip requirements as text found in `site_packages_dir` + as a text. """ - # Also include these packages in the output with --all: wheel, distribute, setuptools, pip + # Also include these packages in the output with --all: wheel, distribute, + # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") + + +comparators = ( + "===", + "~=", + "!=", + "==", + "<=", + ">=", + ">", + "<", +) + +_comparators_re = r"|".join(comparators) +version_splitter = re.compile(rf"({_comparators_re})") + + +def split_req(req): + """ + Return a three-tuple of (name, comparator, version) given a ``req`` + requirement specifier string. Each segment may be empty. Spaces are removed. + + For example: + >>> assert split_req("foo==1.2.3") == ("foo", "==", "1.2.3"), split_req("foo==1.2.3") + >>> assert split_req("foo") == ("foo", "", ""), split_req("foo") + >>> assert split_req("==1.2.3") == ("", "==", "1.2.3"), split_req("==1.2.3") + >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") + >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") + """ + assert req + # do not allow multiple constraints and tags + assert not any(c in req for c in ",;") + req = "".join(req.split()) + if not any(c in req for c in comparators): + return req, "", "" + segments = version_splitter.split(req, maxsplit=1) + return tuple(segments) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index a2fbe4e..e303053 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -11,12 +11,10 @@ from collections import defaultdict import email import itertools -import operator import os import re import shutil import subprocess -import tarfile import tempfile import time import urllib @@ -26,29 +24,30 @@ import packageurl import requests import saneyaml -import utils_pip_compatibility_tags -import utils_pypi_supported_tags - from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packaging import tags as packaging_tags from packaging import version as packaging_version +from urllib.parse import quote_plus + +import utils_pip_compatibility_tags from utils_requirements import load_requirements """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. -- update pip requirement files from installed packages for prod. and dev. -- build and save wheels for all required packages -- also build variants for wheels with native code for all each supported - operating systems (Linux, macOS, Windows) and Python versions (3.x) - combinations using remote Ci jobs -- collect source distributions for all required packages -- keep in sync wheels, distributions, ABOUT and LICENSE files to a PyPI-like - repository (using GitHub) -- create, update and fetch ABOUT, NOTICE and LICENSE metadata for all distributions +- download wheels for packages for all each supported operating systems + (Linux, macOS, Windows) and Python versions (3.x) combinations + +- download sources for packages (aka. sdist) + +- create, update and download ABOUT, NOTICE and LICENSE metadata for these + wheels and source distributions + +- update pip requirement files based on actually installed packages for + production and development Approach @@ -56,35 +55,65 @@ The processing is organized around these key objects: -- A PyPiPackage represents a PyPI package with its name and version. It tracks - the downloadable Distribution objects for that version: +- A PyPiPackage represents a PyPI package with its name and version and the + metadata used to populate an .ABOUT file and document origin and license. + It contains the downloadable Distribution objects for that version: - - one Sdist source Distribution object - - a list of Wheel binary Distribution objects + - one Sdist source Distribution + - a list of Wheel binary Distribution - A Distribution (either a Wheel or Sdist) is identified by and created from its - filename. It also has the metadata used to populate an .ABOUT file and - document origin and license. A Distribution can be fetched from Repository. - Metadata can be loaded from and dumped to ABOUT files and optionally from - DejaCode package data. + filename as well as its name and version. + A Distribution is fetched from a Repository. + Distribution metadata can be loaded from and dumped to ABOUT files. + +- A Wheel binary Distribution can have Python/Platform/OS tags it supports and + was built for and these tags can be matched to an Environment. + +- An Environment is a combination of a Python version and operating system + (e.g., platfiorm and ABI tags.) and is represented by the "tags" it supports. + +- A plain LinksRepository which is just a collection of URLs scrape from a web + page such as HTTP diretory listing. It is used either with pip "--find-links" + option or to fetch ABOUT and LICENSE files. + +- A PypiSimpleRepository is a PyPI "simple" index where a HTML page is listing + package name links. Each such link points to an HTML page listing URLs to all + wheels and sdsist of all versions of this package. + +PypiSimpleRepository and Packages are related through packages name, version and +filenames. + +The Wheel models code is partially derived from the mit-licensed pip and the +Distribution/Wheel/Sdist design has been heavily inspired by the packaging- +dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung +""" + +""" +Wheel downloader -- An Environment is a combination of a Python version and operating system. - A Wheel Distribution also has Python/OS tags is supports and these can be - supported in a given Environment. +- parse requirement file +- create a TODO queue of requirements to process +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} -- Paths or URLs to "filenames" live in a Repository, either a plain - LinksRepository (an HTML page listing URLs or a local directory) or a - PypiRepository (a PyPI simple index where each package name has an HTML page - listing URLs to all distribution types and versions). - Repositories and Distributions are related through filenames. + +- while we have package reqs in TODO queue, process one requirement: + - for each PyPI simple index: + - fetch through cache the PyPI simple index for this package + - for each environment: + - find a wheel matching pinned requirement in this index + - if file exist locally, continue + - fetch the wheel for env + - IF pure, break, no more needed for env + - collect requirement deps from wheel metadata and add to queue + - if fetched, break, otherwise display error message - The Wheel models code is partially derived from the mit-licensed pip and the - Distribution/Wheel/Sdist design has been heavily inspired by the packaging- - dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung """ -TRACE = False +TRACE = True +TRACE_DEEP = False +TRACE_ULTRA_DEEP = False # Supported environments PYTHON_VERSIONS = "36", "37", "38", "39", "310" @@ -106,16 +135,11 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "36": ["cp36", "cp36m"], - "37": ["cp37", "cp37m"], - "38": ["cp38", "cp38m"], - "39": ["cp39", "cp39m"], - "310": ["cp310", "cp310m"], - "36": ["cp36", "abi3"], - "37": ["cp37", "abi3"], - "38": ["cp38", "abi3"], - "39": ["cp39", "abi3"], - "310": ["cp310", "abi3"], + "36": ["cp36", "cp36m", "abi3"], + "37": ["cp37", "cp37m", "abi3"], + "38": ["cp38", "cp38m", "abi3"], + "39": ["cp39", "cp39m", "abi3"], + "310": ["cp310", "cp310m", "abi3"], } PLATFORMS_BY_OS = { @@ -154,7 +178,13 @@ def get_python_dot_version(version): THIRDPARTY_DIR = "thirdparty" CACHE_THIRDPARTY_DIR = ".cache/thirdparty" -REMOTE_LINKS_URL = "https://thirdparty.aboutcode.org/pypi" +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" + +ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" +ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" + +PYPI_SIMPLE_URL = "https://pypi.org/simple" +PYPI_INDEXES = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) EXTENSIONS_APP = (".pyz",) EXTENSIONS_SDIST = ( @@ -171,170 +201,134 @@ def get_python_dot_version(version): ) EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP -PYPI_SIMPLE_URL = "https://pypi.org/simple" - LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org" LICENSING = license_expression.Licensing() -# time to wait build for in seconds, as a string -# 0 measn no wait -DEFAULT_ROMP_BUILD_WAIT = "5" +collect_urls = re.compile('href="([^"]+)"').findall ################################################################################ -# -# Fetch remote wheels and sources locally -# +# Fetch wheels and sources locally ################################################################################ -def fetch_wheels( - environment=None, - requirements_file="requirements.txt", - allow_unpinned=False, +class DistributionNotFound(Exception): + pass + + +def download_wheel( + name, + version, + environment, dest_dir=THIRDPARTY_DIR, - remote_links_url=REMOTE_LINKS_URL, + index_urls=PYPI_INDEXES, ): """ - Download all of the wheel of packages listed in the ``requirements_file`` - requirements file into ``dest_dir`` directory. + Download the wheels binary distribution(s) of package ``name`` and + ``version`` matching the ``environment`` Environment constraints from the + PyPI simple repository ``index_urls`` list of URLs into the ``dest_dir`` + directory. - Only get wheels for the ``environment`` Enviromnent constraints. If the - provided ``environment`` is None then the current Python interpreter - environment is used implicitly. + Raise a DistributionNotFound if no wheel is not found. Otherwise, return a + tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) + """ + if TRACE_DEEP: + print(f" download_wheel: {name}=={version}: {environment}") - Only accept pinned requirements (e.g. with a version) unless - ``allow_unpinned`` is True. + fetched_wheel_filenames = [] + existing_wheel_filenames = [] + try: + for pypi_package in get_package_versions( + name=name, + version=version, + index_urls=index_urls, + ): + if not pypi_package.wheels: + continue - Use exclusively direct downloads from a remote repo at URL - ``remote_links_url``. If ``remote_links_url`` is a path, use this as a - directory of links instead of a URL. + supported_wheels = list(pypi_package.get_supported_wheels(environment=environment)) + if not supported_wheels: + continue - Yield tuples of (PypiPackage, error) where is None on success. - """ - missed = [] + for wheel in supported_wheels: + if os.path.exists(os.path.join(dest_dir, wheel.filename)): + # do not refetch + existing_wheel_filenames.append(wheel.filename) + continue - if not allow_unpinned: - force_pinned = True - else: - force_pinned = False + if TRACE: + print(f" Fetching wheel from index: {wheel.download_url}") + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.add(fetched_wheel_filename) - try: - rrp = list( - get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) except Exception as e: - raise Exception( - dict( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) from e - - fetched_filenames = set() - for name, version, package in rrp: - if not package: - missed.append( - ( - name, - version, - ) - ) - nv = f"{name}=={version}" if version else name - yield None, f"fetch_wheels: Missing package in remote repo: {nv}" + raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: {e}") from e - else: - fetched_filename = package.fetch_wheel( - environment=environment, - fetched_filenames=fetched_filenames, - dest_dir=dest_dir, - ) + if not fetched_wheel_filenames and not existing_wheel_filenames: + raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: No wheel found") - if fetched_filename: - fetched_filenames.add(fetched_filename) - error = None - else: - if fetched_filename in fetched_filenames: - error = None - else: - error = f"Failed to fetch" - yield package, error - - if missed: - rr = get_remote_repo() - print() - print(f"===> fetch_wheels: Missed some packages") - for n, v in missed: - nv = f"{n}=={v}" if v else n - print(f"Missed package {nv} in remote repo, has only:") - for pv in rr.get_versions(n): - print(" ", pv) - raise Exception("Missed some packages in remote repo") + return fetched_wheel_filenames, existing_wheel_filenames -def fetch_sources( - requirements_file="requirements.txt", - allow_unpinned=False, +def download_sdist( + name, + version, dest_dir=THIRDPARTY_DIR, - remote_links_url=REMOTE_LINKS_URL, + index_urls=PYPI_INDEXES, ): """ - Download all of the dependent package sources listed in the - ``requirements_file`` requirements file into ``dest_dir`` destination - directory. - - Use direct downloads to achieve this (not pip download). Use exclusively the - packages found from a remote repo at URL ``remote_links_url``. If - ``remote_links_url`` is a path, use this as a directory of links instead of - a URL. + Download the sdist source distribution of package ``name`` and ``version`` + from the PyPI simple repository ``index_urls`` list of URLs into the + ``dest_dir`` directory. - Only accept pinned requirements (e.g. with a version) unless - ``allow_unpinned`` is True. - - Yield tuples of (PypiPackage, error message) for each package where error - message will empty on success. + Raise a DistributionNotFound if this was not found. Return the filename if + downloaded and False if not downloaded because it already exists. """ - missed = [] + if TRACE_DEEP: + print(f"download_sdist: {name}=={version}: ") - if not allow_unpinned: - force_pinned = True - else: - force_pinned = False + try: + for pypi_package in get_package_versions( + name=name, + version=version, + index_urls=index_urls, + ): + if not pypi_package.sdist: + continue - rrp = list( - get_required_remote_packages( - requirements_file=requirements_file, - force_pinned=force_pinned, - remote_links_url=remote_links_url, - ) - ) + if os.path.exists(os.path.join(dest_dir, pypi_package.sdist.filename)): + # do not refetch + return False + if TRACE: + print(f" Fetching sources from index: {pypi_package.sdist.download_url}") + fetched = pypi_package.sdist.download(dest_dir=dest_dir) + if fetched: + return pypi_package.sdist.filename - for name, version, package in rrp: - if not package: - missed.append( - ( - name, - name, - ) - ) - nv = f"{name}=={version}" if version else name - yield None, f"fetch_sources: Missing package in remote repo: {nv}" + except Exception as e: + raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: {e}") from e - elif not package.sdist: - yield package, f"Missing sdist in links" + raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") - else: - fetched = package.fetch_sdist(dest_dir=dest_dir) - error = f"Failed to fetch" if not fetched else None - yield package, error - if missed: - raise Exception(f"Missing source packages in {remote_links_url}", missed) +def get_package_versions( + name, + version=None, + index_urls=PYPI_INDEXES, +): + """ + Yield PypiPackages with ``name`` and ``version`` from the PyPI simple + repository ``index_urls`` list of URLs. + If ``version`` is not provided, return the latest available versions. + """ + for index_url in index_urls: + try: + repo = get_pypi_repo(index_url) + package = repo.get_package(name, version) + if package: + yield package + except RemoteNotFetchedException as e: + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ # @@ -387,7 +381,7 @@ def sortable_name_version(self): @classmethod def sorted(cls, namevers): - return sorted(namevers, key=cls.sortable_name_version) + return sorted(namevers or [], key=cls.sortable_name_version) @attr.attributes @@ -411,13 +405,6 @@ class Distribution(NameVer): metadata=dict(help="File name."), ) - path_or_url = attr.ib( - repr=False, - type=str, - default="", - metadata=dict(help="Path or download URL."), - ) - sha256 = attr.ib( repr=False, type=str, @@ -546,21 +533,60 @@ def package_url(self): @property def download_url(self): - if self.path_or_url and self.path_or_url.startswith("https://"): - return self.path_or_url - else: - return self.get_best_download_url() + return self.get_best_download_url() + + def get_best_download_url( + self, + index_urls=tuple([PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL]), + ): + """ + Return the best download URL for this distribution where best means that + PyPI is better and our selfhosted repo URLs are second. + If none is found, return a synthetic remote URL. + """ + for index_url in index_urls: + pypi_package = get_pypi_package( + name=self.normalized_name, + version=self.version, + index_url=index_url, + ) + if pypi_package: + if isinstance(pypi_package, tuple): + raise Exception("############", repr(pypi_package)) + try: + pypi_url = pypi_package.get_url_for_filename(self.filename) + except Exception as e: + raise Exception(repr(pypi_package)) from e + if pypi_url: + return pypi_url + + def download(self, dest_dir=THIRDPARTY_DIR): + """ + Download this distribution into `dest_dir` directory. + Return the fetched filename. + """ + assert self.filename + if TRACE: + print( + f"Fetching distribution of {self.name}=={self.version}:", + self.filename, + ) + + fetch_and_save_path_or_url( + filename=self.filename, + dest_dir=dest_dir, + path_or_url=self.path_or_url, + as_text=False, + ) + return self.filename @property def about_filename(self): return f"{self.filename}.ABOUT" - def has_about_file(self, dest_dir=THIRDPARTY_DIR): - return os.path.exists(os.path.join(dest_dir, self.about_filename)) - @property def about_download_url(self): - return self.build_remote_download_url(self.about_filename) + return f"{ABOUT_BASE_URL}/{self.about_filename}" @property def notice_filename(self): @@ -568,7 +594,7 @@ def notice_filename(self): @property def notice_download_url(self): - return self.build_remote_download_url(self.notice_filename) + return f"{ABOUT_BASE_URL}/{self.notice_filename}" @classmethod def from_path_or_url(cls, path_or_url): @@ -601,81 +627,10 @@ def from_filename(cls, filename): Return a distribution built from the data found in a `filename` string. Raise an exception if this is not a valid filename """ + filename = os.path.basename(filename.strip("/")) clazz = cls.get_dist_class(filename) return clazz.from_filename(filename) - @classmethod - def from_data(cls, data, keep_extra=False): - """ - Return a distribution built from a `data` mapping. - """ - filename = data["filename"] - dist = cls.from_filename(filename) - dist.update(data, keep_extra=keep_extra) - return dist - - @classmethod - def from_dist(cls, data, dist): - """ - Return a distribution built from a `data` mapping and update it with data - from another dist Distribution. Return None if it cannot be created - """ - # We can only create from a dist of the same package - has_same_key_fields = all( - data.get(kf) == getattr(dist, kf, None) for kf in ("type", "namespace", "name") - ) - if not has_same_key_fields: - print( - f"Missing key fields: Cannot derive a new dist from data: {data} and dist: {dist}" - ) - return - - has_key_field_values = all(data.get(kf) for kf in ("type", "name", "version")) - if not has_key_field_values: - print( - f"Missing key field values: Cannot derive a new dist from data: {data} and dist: {dist}" - ) - return - - data = dict(data) - # do not overwrite the data with the other dist - # only supplement - data.update({k: v for k, v in dist.get_updatable_data().items() if not data.get(k)}) - return cls.from_data(data) - - @classmethod - def build_remote_download_url(cls, filename, base_url=REMOTE_LINKS_URL): - """ - Return a direct download URL for a file in our remote repo - """ - return f"{base_url}/{filename}" - - def get_best_download_url(self): - """ - Return the best download URL for this distribution where best means that - PyPI is better and our own remote repo URLs are second. - If none is found, return a synthetic remote URL. - """ - name = self.normalized_name - version = self.version - filename = self.filename - - pypi_package = get_pypi_package(name=name, version=version) - if pypi_package: - pypi_url = pypi_package.get_url_for_filename(filename) - if pypi_url: - return pypi_url - - remote_package = get_remote_package(name=name, version=version) - if remote_package: - remote_url = remote_package.get_url_for_filename(filename) - if remote_url: - return remote_url - else: - # the package may not have been published yet, so we craft a URL - # using our remote base URL - return self.build_remote_download_url(self.filename) - def purl_identifiers(self, skinny=False): """ Return a mapping of non-empty identifier name/values for the purl @@ -781,9 +736,11 @@ def save_if_modified(location, content): fo.write(content) return True + as_about = self.to_about() + save_if_modified( location=os.path.join(dest_dir, self.about_filename), - content=saneyaml.dump(self.to_about()), + content=saneyaml.dump(as_about), ) notice_text = self.notice_text and self.notice_text.strip() @@ -844,7 +801,10 @@ def load_remote_about_data(self): NOTICE file if any. Return True if the data was updated. """ try: - about_text = fetch_content_from_path_or_url_through_cache(self.about_download_url) + about_text = fetch_content_from_path_or_url_through_cache( + path_or_url=self.about_download_url, + as_text=True, + ) except RemoteNotFetchedException: return False @@ -855,7 +815,10 @@ def load_remote_about_data(self): notice_file = about_data.pop("notice_file", None) if notice_file: try: - notice_text = fetch_content_from_path_or_url_through_cache(self.notice_download_url) + notice_text = fetch_content_from_path_or_url_through_cache( + path_or_url=self.notice_download_url, + as_text=True, + ) if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: @@ -892,26 +855,23 @@ def validate_checksums(self, dest_dir=THIRDPARTY_DIR): return False return True - def get_pip_hash(self): - """ - Return a pip hash option string as used in requirements for this dist. - """ - assert self.sha256, f"Missinh SHA256 for dist {self}" - return f"--hash=sha256:{self.sha256}" - def get_license_keys(self): try: - keys = LICENSING.license_keys(self.license_expression, unique=True, simple=True) + keys = LICENSING.license_keys( + self.license_expression, + unique=True, + simple=True, + ) except license_expression.ExpressionParseError: return ["unknown"] return keys def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): """ - Fetch license files is missing in `dest_dir`. + Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - paths_or_urls = get_remote_repo().links + urls = LinksRepository.from_url().links errors = [] extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -924,7 +884,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): try: # try remotely first - lic_url = get_link_for_filename(filename=filename, paths_or_urls=paths_or_urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) fetch_and_save_path_or_url( filename=filename, @@ -960,9 +920,17 @@ def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): Return the text of the first PKG-INFO or METADATA file found in the archive of this Distribution in `dest_dir`. Return None if not found. """ - fmt = "zip" if self.filename.endswith(".whl") else None - dist = os.path.join(dest_dir, self.filename) - with tempfile.TemporaryDirectory(prefix="pypi-tmp-extract") as td: + + fn = self.filename + if fn.endswith(".whl"): + fmt = "zip" + elif fn.endswith(".tar.gz"): + fmt = "gztar" + else: + fmt = None + + dist = os.path.join(dest_dir, fn) + with tempfile.TemporaryDirectory(prefix=f"pypi-tmp-extract-{fn}") as td: shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) # NOTE: we only care about the first one found in the dist # which may not be 100% right @@ -983,7 +951,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): """ pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) if not pkginfo_text: - print(f"!!!!PKG-INFO not found in {self.filename}") + print(f"!!!!PKG-INFO/METADATA not found in {self.filename}") return raw_data = email.message_from_string(pkginfo_text) @@ -1075,6 +1043,20 @@ def update(self, data, overwrite=False, keep_extra=True): return updated +def get_license_link_for_filename(filename, urls): + """ + Return a link for `filename` found in the `links` list of URLs or paths. Raise an + exception if no link is found or if there are more than one link for that + file name. + """ + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + if not path_or_url: + raise Exception(f"Missing link to file: {filename}") + if not len(path_or_url) == 1: + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + return path_or_url[0] + + class InvalidDistributionFilename(Exception): pass @@ -1243,15 +1225,12 @@ def is_supported_by_tags(self, tags): """ Return True is this wheel is compatible with one of a list of PEP 425 tags. """ + if TRACE_DEEP: + print() + print("is_supported_by_tags: tags:", tags) + print("self.tags:", self.tags) return not self.tags.isdisjoint(tags) - def is_supported_by_environment(self, environment): - """ - Return True if this wheel is compatible with the Environment - `environment`. - """ - return not self.is_supported_by_tags(environment.tags) - def to_filename(self): """ Return a wheel filename reconstructed from its fields (that may not be @@ -1306,8 +1285,8 @@ class PypiPackage(NameVer): sdist = attr.ib( repr=False, - type=str, - default="", + type=Sdist, + default=None, metadata=dict(help="Sdist source distribution for this package."), ) @@ -1328,22 +1307,14 @@ def specifier(self): else: return self.name - @property - def specifier_with_hashes(self): - """ - Return a requirement specifier for this package with --hash options for - all its distributions - """ - items = [self.specifier] - items += [d.get_pip_hashes() for d in self.get_distributions()] - return " \\\n ".join(items) - - def get_supported_wheels(self, environment): + def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): """ Yield all the Wheel of this package supported and compatible with the Environment `environment`. """ envt_tags = environment.tags() + if verbose: + print("get_supported_wheels: envt_tags:", envt_tags) for wheel in self.wheels: if wheel.is_supported_by_tags(envt_tags): yield wheel @@ -1369,6 +1340,8 @@ def package_from_dists(cls, dists): >>> assert package.wheels == [w1, w2] """ dists = list(dists) + if TRACE_DEEP: + print(f"package_from_dists: {dists}") if not dists: return @@ -1379,13 +1352,21 @@ def package_from_dists(cls, dists): package = PypiPackage(name=normalized_name, version=version) for dist in dists: - if dist.normalized_name != normalized_name or dist.version != version: + if dist.normalized_name != normalized_name: if TRACE: print( - f" Skipping inconsistent dist name and version: {dist} " - f'Expected instead package name: {normalized_name} and version: "{version}"' + f" Skipping inconsistent dist name: expected {normalized_name} got {dist}" ) continue + elif dist.version != version: + dv = packaging_version.parse(dist.version) + v = packaging_version.parse(version) + if dv != v: + if TRACE: + print( + f" Skipping inconsistent dist version: expected {version} got {dist}" + ) + continue if isinstance(dist, Sdist): package.sdist = dist @@ -1396,39 +1377,41 @@ def package_from_dists(cls, dists): else: raise Exception(f"Unknown distribution type: {dist}") + if TRACE_DEEP: + print(f"package_from_dists: {package}") + return package @classmethod - def packages_from_one_path_or_url(cls, path_or_url): + def packages_from_dir(cls, directory): """ - Yield PypiPackages built from files found in at directory path or the - URL to an HTML page (that will be fetched). + Yield PypiPackages built from files found in at directory path. """ - extracted_paths_or_urls = get_paths_or_urls(path_or_url) - return cls.packages_from_many_paths_or_urls(extracted_paths_or_urls) + base = os.path.abspath(directory) + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + if TRACE_ULTRA_DEEP: + print("packages_from_dir: paths:", paths) + return cls.packages_from_many_paths_or_urls(paths) @classmethod def packages_from_many_paths_or_urls(cls, paths_or_urls): """ Yield PypiPackages built from a list of paths or URLs. """ - dists = cls.get_dists(paths_or_urls) + dists = cls.dists_from_paths_or_urls(paths_or_urls) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls: dists:", dists) + dists = NameVer.sorted(dists) for _projver, dists_of_package in itertools.groupby( dists, key=NameVer.sortable_name_version, ): - yield PypiPackage.package_from_dists(dists_of_package) - - @classmethod - def get_versions_from_path_or_url(cls, name, path_or_url): - """ - Return a subset list from a list of PypiPackages version at `path_or_url` - that match PypiPackage `name`. - """ - packages = cls.packages_from_one_path_or_url(path_or_url) - return cls.get_versions(name, packages) + package = PypiPackage.package_from_dists(dists_of_package) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls", package) + yield package @classmethod def get_versions(cls, name, packages): @@ -1451,15 +1434,6 @@ def get_latest_version(cls, name, packages): return return versions[-1] - @classmethod - def get_outdated_versions(cls, name, packages): - """ - Return all versions except the latest version of PypiPackage `name` from a - list of `packages`. - """ - versions = cls.get_versions(name, packages) - return versions[:-1] - @classmethod def get_name_version(cls, name, version, packages): """ @@ -1467,100 +1441,23 @@ def get_name_version(cls, name, version, packages): or None if it is not found. If `version` is None, return the latest version found. """ - if version is None: + if TRACE_ULTRA_DEEP: + print("get_name_version:", name, version, packages) + if not version: return cls.get_latest_version(name, packages) nvs = [p for p in cls.get_versions(name, packages) if p.version == version] if not nvs: - return + return name, version if len(nvs) == 1: return nvs[0] raise Exception(f"More than one PypiPackage with {name}=={version}") - def fetch_wheel( - self, - environment=None, - fetched_filenames=None, - dest_dir=THIRDPARTY_DIR, - ): - """ - Download a binary wheel of this package matching the ``environment`` - Enviromnent constraints into ``dest_dir`` directory. - - Return the wheel filename if it was fetched, None otherwise. - - If the provided ``environment`` is None then the current Python - interpreter environment is used implicitly. Do not refetch wheel if - their name is in a provided ``fetched_filenames`` set. - """ - fetched_wheel_filename = None - if fetched_filenames is not None: - fetched_filenames = fetched_filenames - else: - fetched_filenames = set() - - supported_wheels = list(self.get_supported_wheels(environment)) - for wheel in supported_wheels: - - if wheel.filename not in fetched_filenames: - fetch_and_save_path_or_url( - filename=wheel.filename, - path_or_url=wheel.path_or_url, - dest_dir=dest_dir, - as_text=False, - ) - fetched_filenames.add(wheel.filename) - fetched_wheel_filename = wheel.filename - - # TODO: what if there is more than one? - break - - return fetched_wheel_filename - - def fetch_sdist(self, dest_dir=THIRDPARTY_DIR): - """ - Download the source distribution into `dest_dir` directory. Return the - fetched filename if it was fetched, False otherwise. - """ - if self.sdist: - assert self.sdist.filename - if TRACE: - print("Fetching source for package:", self.name, self.version) - fetch_and_save_path_or_url( - filename=self.sdist.filename, - dest_dir=dest_dir, - path_or_url=self.sdist.path_or_url, - as_text=False, - ) - if TRACE: - print(" --> file:", self.sdist.filename) - return self.sdist.filename - else: - print(f"Missing sdist for: {self.name}=={self.version}") - return False - - def delete_files(self, dest_dir=THIRDPARTY_DIR): - """ - Delete all PypiPackage files from `dest_dir` including wheels, sdist and - their ABOUT files. Note that we do not delete licenses since they can be - shared by several packages: therefore this would be done elsewhere in a - function that is aware of all used licenses. - """ - for to_delete in self.wheels + [self.sdist]: - if not to_delete: - continue - tdfn = to_delete.filename - for deletable in [tdfn, f"{tdfn}.ABOUT", f"{tdfn}.NOTICE"]: - target = os.path.join(dest_dir, deletable) - if os.path.exists(target): - print(f"Deleting outdated {target}") - fileutils.delete(target) - @classmethod - def get_dists(cls, paths_or_urls): + def dists_from_paths_or_urls(cls, paths_or_urls): """ Return a list of Distribution given a list of `paths_or_urls` to wheels or source distributions. @@ -1574,9 +1471,9 @@ def get_dists(cls, paths_or_urls): ... /home/foo/bitarray-0.8.1-cp36-cp36m-linux_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl - ... httsp://example.com/bar/bitarray-0.8.1.tar.gz + ... https://example.com/bar/bitarray-0.8.1.tar.gz ... bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split() - >>> result = list(PypiPackage.get_dists(paths_or_urls)) + >>> result = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) >>> for r in results: ... r.filename = '' ... r.path_or_url = '' @@ -1590,18 +1487,28 @@ def get_dists(cls, paths_or_urls): ... Wheel(name='bitarray', version='0.8.1', build='', ... python_versions=['cp36'], abis=['cp36m'], ... platforms=['win_amd64']), + ... Sdist(name='bitarray', version='0.8.1'), ... Sdist(name='bitarray', version='0.8.1') ... ] >>> assert expected == result """ + dists = [] + if TRACE_DEEP: + print(" ###paths_or_urls:", paths_or_urls) installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: - yield Distribution.from_path_or_url(path_or_url) + dist = Distribution.from_path_or_url(path_or_url) + dists.append(dist) + if TRACE_DEEP: + print( + " ===> dists_from_paths_or_urls:", dist, "with URL:", dist.download_url + ) except InvalidDistributionFilename: - if TRACE: - print(f"Skipping invalid distribution from: {path_or_url}") + if TRACE_DEEP: + print(f" Skipping invalid distribution from: {path_or_url}") continue + return dists def get_distributions(self): """ @@ -1626,10 +1533,11 @@ class Environment: """ An Environment describes a target installation environment with its supported Python version, ABI, platform, implementation and related - attributes. We can use these to pass as `pip download` options and force - fetching only the subset of packages that match these Environment - constraints as opposed to the current running Python interpreter - constraints. + attributes. + + We can use these to pass as `pip download` options and force fetching only + the subset of packages that match these Environment constraints as opposed + to the current running Python interpreter constraints. """ python_version = attr.ib( @@ -1648,18 +1556,21 @@ class Environment: type=str, default="cp", metadata=dict(help="Python implementation supported by this environment."), + repr=False, ) abis = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help="List of ABI tags supported by this environment."), + metadata=dict(help="List of ABI tags supported by this environment."), + repr=False, ) platforms = attr.ib( type=list, default=attr.Factory(list), metadata=dict(help="List of platform tags supported by this environment."), + repr=False, ) @classmethod @@ -1677,18 +1588,20 @@ def from_pyver_and_os(cls, python_version, operating_system): def get_pip_cli_options(self): """ - Return a list of pip command line options for this environment. + Return a list of pip download command line options for this environment. """ options = [ "--python-version", self.python_version, "--implementation", self.implementation, - "--abi", - self.abi, ] + for abi in self.abis: + options.extend(["--abi", abi]) + for platform in self.platforms: options.extend(["--platform", platform]) + return options def tags(self): @@ -1704,7 +1617,6 @@ def tags(self): ) ) - ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1713,11 +1625,18 @@ def tags(self): @attr.attributes -class Repository: +class PypiSimpleRepository: """ - A PyPI or links Repository of Python packages: wheels, sdist, ABOUT, etc. + A PyPI repository of Python packages: wheels, sdist, etc. like the public + PyPI simple index. It is populated lazily based on requested packages names. """ + index_url = attr.ib( + type=str, + default=PYPI_SIMPLE_URL, + metadata=dict(help="Base PyPI simple URL for this index."), + ) + packages_by_normalized_name = attr.ib( type=dict, default=attr.Factory(lambda: defaultdict(list)), @@ -1730,126 +1649,157 @@ class Repository: metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), ) - def get_links(self, *args, **kwargs): - raise NotImplementedError() - def get_versions(self, name): """ Return a list of all available PypiPackage version for this package name. The list may be empty. """ - raise NotImplementedError() + name = name and NameVer.normalize_name(name) + self._populate_links_and_packages(name) + return self.packages_by_normalized_name.get(name, []) + + def get_latest_version(self, name): + """ + Return the latest PypiPackage version for this package name or None. + """ + versions = self.get_versions(name) + return PypiPackage.get_latest_version(name, versions) def get_package(self, name, version): """ Return the PypiPackage with name and version or None. """ - raise NotImplementedError() + versions = self.get_versions(name) + if TRACE_DEEP: + print("PypiPackage.get_package:versions:", versions) + return PypiPackage.get_name_version(name, version, versions) - def get_latest_version(self, name): + def _fetch_links(self, name, _LINKS={}): """ - Return the latest PypiPackage version for this package name or None. + Return a list of download link URLs found in a PyPI simple index for package + name using the `index_url` of this repository. """ - raise NotImplementedError() + name = name and NameVer.normalize_name(name) + index_url = self.index_url + name = name and NameVer.normalize_name(name) + index_url = index_url.strip("/") + index_url = f"{index_url}/{name}" -@attr.attributes -class LinksRepository(Repository): - """ - Represents a simple links repository which is either a local directory with - Python wheels and sdist or a remote URL to an HTML with links to these. - (e.g. suitable for use with pip --find-links). - """ + if TRACE_DEEP: + print( + f" Finding links for {name!r} from PyPI index: {index_url} : cached?:", + index_url in _LINKS, + ) - path_or_url = attr.ib( - type=str, - default="", - metadata=dict(help="Package directory path or URL"), - ) + if index_url not in _LINKS: + text = fetch_content_from_path_or_url_through_cache(path_or_url=index_url, as_text=True) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] + _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] - links = attr.ib( - type=list, - default=attr.Factory(list), - metadata=dict(help="List of links available in this repo"), - ) + links = _LINKS[index_url] + if TRACE_DEEP: + print(f" Found links {links!r}") + return links - def __attrs_post_init__(self): - if not self.links: - self.links = get_paths_or_urls(links_url=self.path_or_url) - if not self.packages_by_normalized_name: - for p in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=self.links): - normalized_name = p.normalized_name - self.packages_by_normalized_name[normalized_name].append(p) - self.packages_by_normalized_name_version[(normalized_name, p.version)] = p + def _populate_links_and_packages(self, name): + name = name and NameVer.normalize_name(name) - def get_links(self, *args, **kwargs): - return self.links or [] + if TRACE_DEEP: + print("PypiPackage._populate_links_and_packages:name:", name) - def get_versions(self, name): - name = name and NameVer.normalize_name(name) - return self.packages_by_normalized_name.get(name, []) + links = self._fetch_links(name) + packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - def get_latest_version(self, name): - return PypiPackage.get_latest_version(name, self.get_versions(name)) + if TRACE_DEEP: + print("PypiPackage._populate_links_and_packages:packages:", packages) - def get_package(self, name, version): - return PypiPackage.get_name_version(name, version, self.get_versions(name)) + self.packages_by_normalized_name[name] = packages + + for p in packages: + name = name and NameVer.normalize_name(p.name) + self.packages_by_normalized_name_version[(name, p.version)] = p @attr.attributes -class PypiRepository(Repository): +class LinksRepository: """ - Represents the public PyPI simple index. - It is populated lazily based on requested packages names + Represents a simple links repository such an HTTP directory listing or a + page with links. """ - simple_url = attr.ib( + url = attr.ib( type=str, - default=PYPI_SIMPLE_URL, - metadata=dict(help="Base PyPI simple URL for this index."), + default="", + metadata=dict(help="Links directory URL"), ) - links_by_normalized_name = attr.ib( - type=dict, - default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help="Mapping of {package name: [links]} available in this repo"), + links = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of links available in this repo"), ) - def _fetch_links(self, name): - name = name and NameVer.normalize_name(name) - return find_pypi_links(name=name, simple_url=self.simple_url) + def __attrs_post_init__(self): + if not self.links: + self.links = self.find_links() - def _populate_links_and_packages(self, name): - name = name and NameVer.normalize_name(name) - if name in self.links_by_normalized_name: - return + def find_links(self): + """ + Return a list of link URLs found in the HTML page at `self.url` + """ + links_url = self.url + if TRACE_DEEP: + print(f"Finding links from: {links_url}") + plinks_url = urllib.parse.urlparse(links_url) + base_url = urllib.parse.SplitResult( + plinks_url.scheme, plinks_url.netloc, "", "", "" + ).geturl() - links = self._fetch_links(name) - self.links_by_normalized_name[name] = links + if TRACE_DEEP: + print(f"Base URL {base_url}") - packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - self.packages_by_normalized_name[name] = packages + text = fetch_content_from_path_or_url_through_cache( + path_or_url=links_url, + as_text=True, + ) - for p in packages: - name = name and NameVer.normalize_name(p.name) - self.packages_by_normalized_name_version[(name, p.version)] = p + links = [] + for link in collect_urls(text): + if not link.endswith(EXTENSIONS): + continue - def get_links(self, name, *args, **kwargs): - name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) - return self.links_by_normalized_name.get(name, []) + plink = urllib.parse.urlsplit(link) - def get_versions(self, name): - name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) - return self.packages_by_normalized_name.get(name, []) + if plink.scheme: + # full URL kept as-is + url = link - def get_latest_version(self, name): - return PypiPackage.get_latest_version(name, self.get_versions(name)) + if plink.path.startswith("/"): + # absolute link + url = f"{base_url}{link}" - def get_package(self, name, version): - return PypiPackage.get_name_version(name, version, self.get_versions(name)) + else: + # relative link + url = f"{links_url}/{link}" + + if TRACE_DEEP: + print(f"Adding URL: {url}") + links.append(url) + + if TRACE: + print(f"Found {len(links)} links at {links_url}") + return links + + @classmethod + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): + if url not in _LINKS_REPO: + _LINKS_REPO[url] = cls(url=url) + return _LINKS_REPO[url] ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the @@ -1862,52 +1812,27 @@ def get_local_packages(directory=THIRDPARTY_DIR): Return the list of all PypiPackage objects built from a local directory. Return an empty list if the package cannot be found. """ - return list(PypiPackage.packages_from_one_path_or_url(path_or_url=directory)) - - -def get_local_repo(directory=THIRDPARTY_DIR): - return LinksRepository(path_or_url=directory) - + return list(PypiPackage.packages_from_dir(directory=directory)) -_REMOTE_REPO = None +def get_pypi_repo(index_url, _PYPI_REPO={}): + if index_url not in _PYPI_REPO: + _PYPI_REPO[index_url] = PypiSimpleRepository(index_url=index_url) + return _PYPI_REPO[index_url] -def get_remote_repo(remote_links_url=REMOTE_LINKS_URL): - global _REMOTE_REPO - if not _REMOTE_REPO: - _REMOTE_REPO = LinksRepository(path_or_url=remote_links_url) - return _REMOTE_REPO - -def get_remote_package(name, version, remote_links_url=REMOTE_LINKS_URL): +def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP): """ Return a PypiPackage or None. """ try: - return get_remote_repo(remote_links_url).get_package(name, version) - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - - -_PYPI_REPO = None - - -def get_pypi_repo(pypi_simple_url=PYPI_SIMPLE_URL): - global _PYPI_REPO - if not _PYPI_REPO: - _PYPI_REPO = PypiRepository(simple_url=pypi_simple_url) - return _PYPI_REPO - + package = get_pypi_repo(index_url).get_package(name, version) + if verbose: + print(f" get_pypi_package: {name} @ {version} info from {index_url}: {package}") + return package -def get_pypi_package(name, version, pypi_simple_url=PYPI_SIMPLE_URL): - """ - Return a PypiPackage or None. - """ - try: - return get_pypi_repo(pypi_simple_url).get_package(name, version) except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ # @@ -1936,8 +1861,8 @@ def get(self, path_or_url, as_text=True): Get a file from a `path_or_url` through the cache. `path_or_url` can be a path or a URL to a file. """ - filename = os.path.basename(path_or_url.strip("/")) - cached = os.path.join(self.directory, filename) + cache_key = quote_plus(path_or_url.strip("/")) + cached = os.path.join(self.directory, cache_key) if not os.path.exists(cached): content = get_file_content(path_or_url=path_or_url, as_text=as_text) @@ -1948,32 +1873,23 @@ def get(self, path_or_url, as_text=True): else: return get_local_file_content(path=cached, as_text=as_text) - def put(self, filename, content): - """ - Put in the cache the `content` of `filename`. - """ - cached = os.path.join(self.directory, filename) - wmode = "wb" if isinstance(content, bytes) else "w" - with open(cached, wmode) as fo: - fo.write(content) - def get_file_content(path_or_url, as_text=True): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ - if path_or_url.startswith("file://") or ( - path_or_url.startswith("/") and os.path.exists(path_or_url) - ): - return get_local_file_content(path=path_or_url, as_text=as_text) - - elif path_or_url.startswith("https://"): + if path_or_url.startswith("https://"): if TRACE: print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content + elif path_or_url.startswith("file://") or ( + path_or_url.startswith("/") and os.path.exists(path_or_url) + ): + return get_local_file_content(path=path_or_url, as_text=as_text) + else: raise Exception(f"Unsupported URL scheme: {path_or_url}") @@ -2016,6 +1932,7 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header + print(f" DOWNLOADING {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -2039,76 +1956,11 @@ def get_remote_file_content( return response.headers, response.text if as_text else response.content -def get_url_content_if_modified( - url, - md5, - _delay=0, -): - """ - Return fetched content bytes at `url` or None if the md5 has not changed. - Retries multiple times to fetch if there is a HTTP 429 throttling response - and this with an increasing delay. - """ - time.sleep(_delay) - headers = None - if md5: - etag = f'"{md5}"' - headers = {"If-None-Match": f"{etag}"} - - # using a GET with stream=True ensure we get the the final header from - # several redirects and that we can ignore content there. A HEAD request may - # not get us this last header - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: - status = response.status_code - if status == requests.codes.too_many_requests and _delay < 20: # NOQA - # too many requests: start waiting with some exponential delay - _delay = (_delay * 2) or 1 - return get_url_content_if_modified(url=url, md5=md5, _delay=_delay) - - elif status == requests.codes.not_modified: # NOQA - # all is well, the md5 is the same - return None - - elif status != requests.codes.ok: # NOQA - raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") - - return response.content - - -def get_remote_headers(url): - """ - Fetch and return a mapping of HTTP headers of `url`. - """ - headers, _content = get_remote_file_content(url, headers_only=True) - return headers - - -def fetch_and_save_filename_from_paths_or_urls( - filename, - paths_or_urls, - dest_dir=THIRDPARTY_DIR, +def fetch_content_from_path_or_url_through_cache( + path_or_url, as_text=True, + cache=Cache(), ): - """ - Return the content from fetching the `filename` file name found in the - `paths_or_urls` list of URLs or paths and save to `dest_dir`. Raise an - Exception on errors. Treats the content as text if `as_text` is True - otherwise as binary. - """ - path_or_url = get_link_for_filename( - filename=filename, - paths_or_urls=paths_or_urls, - ) - - return fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, - path_or_url=path_or_url, - as_text=as_text, - ) - - -def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cache=Cache()): """ Return the content from fetching at path or URL. Raise an Exception on errors. Treats the content as text if as_text is True otherwise as treat as @@ -2118,423 +1970,90 @@ def fetch_content_from_path_or_url_through_cache(path_or_url, as_text=True, cach Note: the `cache` argument is a global, though it does not really matter since it does not hold any state which is only kept on disk. """ - if cache: - return cache.get(path_or_url=path_or_url, as_text=as_text) - else: - return get_file_content(path_or_url=path_or_url, as_text=as_text) + return cache.get(path_or_url=path_or_url, as_text=as_text) -def fetch_and_save_path_or_url(filename, dest_dir, path_or_url, as_text=True, through_cache=True): +def fetch_and_save_path_or_url( + filename, + dest_dir, + path_or_url, + as_text=True, +): """ Return the content from fetching the `filename` file name at URL or path and save to `dest_dir`. Raise an Exception on errors. Treats the content as text if as_text is True otherwise as treat as binary. """ - if through_cache: - content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text) - else: - content = fetch_content_from_path_or_url_through_cache(path_or_url, as_text, cache=None) - + content = fetch_content_from_path_or_url_through_cache( + path_or_url=path_or_url, + as_text=as_text, + ) output = os.path.join(dest_dir, filename) wmode = "w" if as_text else "wb" with open(output, wmode) as fo: fo.write(content) return content - ################################################################################ -# -# Sync and fix local thirdparty directory for various issues and gaps -# +# Requirements processing ################################################################################ -def fetch_missing_sources(dest_dir=THIRDPARTY_DIR): - """ - Given a thirdparty dir, fetch missing source distributions from our remote - repo or PyPI. Return a list of (name, version) tuples for source - distribution that were not found - """ - not_found = [] - local_packages = get_local_packages(directory=dest_dir) - remote_repo = get_remote_repo() - pypi_repo = get_pypi_repo() - - for package in local_packages: - if not package.sdist: - print(f"Finding sources for: {package.name}=={package.version}: ", end="") - try: - pypi_package = pypi_repo.get_package(name=package.name, version=package.version) - - if pypi_package and pypi_package.sdist: - print(f"Fetching sources from Pypi") - pypi_package.fetch_sdist(dest_dir=dest_dir) - continue - else: - remote_package = remote_repo.get_package( - name=package.name, version=package.version - ) - - if remote_package and remote_package.sdist: - print(f"Fetching sources from Remote") - remote_package.fetch_sdist(dest_dir=dest_dir) - continue - - except RemoteNotFetchedException as e: - print(f"Failed to fetch remote package info: {e}") - - print(f"No sources found") - not_found.append( - ( - package.name, - package.version, - ) - ) - - return not_found - - -def fetch_missing_wheels( - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, +def get_required_remote_packages( + requirements_file="requirements.txt", + index_url=PYPI_SIMPLE_URL, ): """ - Given a thirdparty dir fetch missing wheels for all known combos of Python - versions and OS. Return a list of tuple (Package, Environment) for wheels - that were not found locally or remotely. + Yield tuple of (name, version, PypiPackage) for packages listed in the + `requirements_file` requirements file and found in the PyPI index + ``index_url`` URL. """ - local_packages = get_local_packages(directory=dest_dir) - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - packages_and_envts = itertools.product(local_packages, environments) - - not_fetched = [] - fetched_filenames = set() - for package, envt in packages_and_envts: + required_name_versions = load_requirements(requirements_file=requirements_file) + return get_required_packages(required_name_versions=required_name_versions, index_url=index_url) - filename = package.fetch_wheel( - environment=envt, - fetched_filenames=fetched_filenames, - dest_dir=dest_dir, - ) - if filename: - fetched_filenames.add(filename) - else: - not_fetched.append( - ( - package, - envt, - ) - ) - - return not_fetched - - -def build_missing_wheels( - packages_and_envts, - build_remotely=False, - with_deps=False, - dest_dir=THIRDPARTY_DIR, - remote_build_log_file=None, +def get_required_packages( + required_name_versions, + index_url=PYPI_SIMPLE_URL, ): """ - Build all wheels in a list of tuple (Package, Environment) and save in - `dest_dir`. Return a list of tuple (Package, Environment), and a list of - built wheel filenames. - """ - - not_built = [] - built_filenames = [] - - packages_and_envts = itertools.groupby(sorted(packages_and_envts), key=operator.itemgetter(0)) - - for package, pkg_envts in packages_and_envts: - - envts = [envt for _pkg, envt in pkg_envts] - python_versions = sorted(set(e.python_version for e in envts)) - operating_systems = sorted(set(e.operating_system for e in envts)) - built = None - try: - built = build_wheels( - requirements_specifier=package.specifier, - with_deps=with_deps, - build_remotely=build_remotely, - python_versions=python_versions, - operating_systems=operating_systems, - verbose=TRACE, - dest_dir=dest_dir, - remote_build_log_file=remote_build_log_file, - ) - print(".") - except Exception as e: - import traceback - - print("#############################################################") - print("############# WHEEL BUILD FAILED ######################") - traceback.print_exc() - print() - print("#############################################################") - - if not built: - for envt in pkg_envts: - not_built.append((package, envt)) - else: - for bfn in built: - print(f" --> Built wheel: {bfn}") - built_filenames.append(bfn) - - return not_built, built_filenames - - -################################################################################ -# -# Functions to handle remote or local repo used to "find-links" -# -################################################################################ - - -def get_paths_or_urls(links_url): - if links_url.startswith("https:"): - paths_or_urls = find_links_from_release_url(links_url) - else: - paths_or_urls = find_links_from_dir(links_url) - return paths_or_urls - - -def find_links_from_dir(directory=THIRDPARTY_DIR): - """ - Return a list of path to files in `directory` for any file that ends with - any of the extension in the list of `extensions` strings. - """ - base = os.path.abspath(directory) - files = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] - return files - - -get_links = re.compile('href="([^"]+)"').findall - - -def find_links_from_release_url(links_url=REMOTE_LINKS_URL): - """ - Return a list of download link URLs found in the HTML page at `links_url` - URL that starts with the `prefix` string and ends with any of the extension - in the list of `extensions` strings. Use the `base_url` to prefix the links. + Yield tuple of (name, version) or a PypiPackage for package name/version + listed in the ``required_name_versions`` list and found in the PyPI index + ``index_url`` URL. """ if TRACE: - print(f"Finding links for {links_url}") - - plinks_url = urllib.parse.urlparse(links_url) - - base_url = urllib.parse.SplitResult(plinks_url.scheme, plinks_url.netloc, "", "", "").geturl() - - if TRACE: - print(f"Base URL {base_url}") - - _headers, text = get_remote_file_content(links_url) - links = [] - for link in get_links(text): - if not link.endswith(EXTENSIONS): - continue + print("get_required_packages", index_url) - plink = urllib.parse.urlsplit(link) - - if plink.scheme: - # full URL kept as-is - url = link - - if plink.path.startswith("/"): - # absolute link - url = f"{base_url}{link}" - - else: - # relative link - url = f"{links_url}/{link}" + repo = get_pypi_repo(index_url=index_url) + for name, version in required_name_versions: if TRACE: - print(f"Adding URL: {url}") - - links.append(url) - - if TRACE: - print(f"Found {len(links)} links at {links_url}") - return links - - -def find_pypi_links(name, simple_url=PYPI_SIMPLE_URL): - """ - Return a list of download link URLs found in a PyPI simple index for package name. - with the list of `extensions` strings. Use the `simple_url` PyPI url. - """ - if TRACE: - print(f"Finding links for {simple_url}") - - name = name and NameVer.normalize_name(name) - simple_url = simple_url.strip("/") - simple_url = f"{simple_url}/{name}" - - _headers, text = get_remote_file_content(simple_url) - links = get_links(text) - # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] - links = [url for url, _, _sha256 in links] - links = [l for l in links if l.endswith(EXTENSIONS)] - return links - - -def get_link_for_filename(filename, paths_or_urls): - """ - Return a link for `filename` found in the `links` list of URLs or paths. Raise an - exception if no link is found or if there are more than one link for that - file name. - """ - path_or_url = [l for l in paths_or_urls if l.endswith(f"/{filename}")] - if not path_or_url: - raise Exception(f"Missing link to file: {filename}") - if not len(path_or_url) == 1: - raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) - return path_or_url[0] - + print(" get_required_packages: name:", name, "version:", version) + yield repo.get_package(name, version) ################################################################################ -# -# Requirements processing -# +# Functions to update or fetch ABOUT and license files ################################################################################ -class MissingRequirementException(Exception): - pass - - -def get_required_packages(required_name_versions): - """ - Return a tuple of (remote packages, PyPI packages) where each is a mapping - of {(name, version): PypiPackage} for packages listed in the - `required_name_versions` list of (name, version) tuples. Raise a - MissingRequirementException with a list of missing (name, version) if a - requirement cannot be satisfied remotely or in PyPI. - """ - remote_repo = get_remote_repo() - - remote_packages = { - (name, version): remote_repo.get_package(name, version) - for name, version in required_name_versions - } - - pypi_repo = get_pypi_repo() - pypi_packages = { - (name, version): pypi_repo.get_package(name, version) - for name, version in required_name_versions - } - - # remove any empty package (e.g. that do not exist in some place) - remote_packages = {nv: p for nv, p in remote_packages.items() if p} - pypi_packages = {nv: p for nv, p in pypi_packages.items() if p} - - # check that we are not missing any - repos_name_versions = set(remote_packages.keys()) | set(pypi_packages.keys()) - missing_name_versions = required_name_versions.difference(repos_name_versions) - if missing_name_versions: - raise MissingRequirementException(sorted(missing_name_versions)) - - return remote_packages, pypi_packages - - -def get_required_remote_packages( - requirements_file="requirements.txt", - force_pinned=True, - remote_links_url=REMOTE_LINKS_URL, +def clean_about_files( + dest_dir=THIRDPARTY_DIR, ): """ - Yield tuple of (name, version, PypiPackage) for packages listed in the - `requirements_file` requirements file and found in the PyPI-like link repo - ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` as a - local directory path to a wheels directory if this is not a a URL. - """ - required_name_versions = load_requirements( - requirements_file=requirements_file, - force_pinned=force_pinned, - ) - - if remote_links_url.startswith("https://"): - repo = get_remote_repo(remote_links_url=remote_links_url) - else: - # a local path - assert os.path.exists(remote_links_url), f"Path does not exist: {remote_links_url}" - repo = get_local_repo(directory=remote_links_url) - - for name, version in required_name_versions: - if version: - yield name, version, repo.get_package(name, version) - else: - yield name, version, repo.get_latest_version(name) - - -def update_requirements(name, version=None, requirements_file="requirements.txt"): - """ - Upgrade or add `package_name` with `new_version` to the `requirements_file` - requirements file. Write back requirements sorted with name and version - canonicalized. Note: this cannot deal with hashed or unpinned requirements. - Do nothing if the version already exists as pinned. + Given a thirdparty dir, clean ABOUT files """ - normalized_name = NameVer.normalize_name(name) - - is_updated = False - updated_name_versions = [] - for existing_name, existing_version in load_requirements(requirements_file, force_pinned=False): - - existing_normalized_name = NameVer.normalize_name(existing_name) - - if normalized_name == existing_normalized_name: - if version != existing_version: - is_updated = True - updated_name_versions.append( - ( - existing_normalized_name, - existing_version, - ) - ) - - if is_updated: - updated_name_versions = sorted(updated_name_versions) - nvs = "\n".join(f"{name}=={version}" for name, version in updated_name_versions) - - with open(requirements_file, "w") as fo: - fo.write(nvs) - - -def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.txt"): - """ - Hash all the requirements found in the `requirements_file` - requirements file based on distributions available in `dest_dir` - """ - local_repo = get_local_repo(directory=dest_dir) - packages_by_normalized_name_version = local_repo.packages_by_normalized_name_version - hashed = [] - for name, version in load_requirements(requirements_file, force_pinned=True): - package = packages_by_normalized_name_version.get((name, version)) - if not package: - raise Exception(f"Missing required package {name}=={version}") - hashed.append(package.specifier_with_hashes) - - with open(requirements_file, "w") as fo: - fo.write("\n".join(hashed)) - + local_packages = get_local_packages(directory=dest_dir) + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) -################################################################################ -# -# Functions to update or fetch ABOUT and license files -# -################################################################################ + if "classifiers" in local_dist.extra_data: + local_dist.extra_data.pop("classifiers", None) + local_dist.save_about_and_notice_files(dest_dir) -def add_fetch_or_update_about_and_license_files( - dest_dir=THIRDPARTY_DIR, - include_remote=True, - strip_classifiers=False, -): +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): """ Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using best efforts: @@ -2544,32 +2063,28 @@ def add_fetch_or_update_about_and_license_files( - derive from existing distribution with same name and latest version that would have such ABOUT file - extract ABOUT file data from distributions PKGINFO or METADATA files - - TODO: make API calls to fetch package data from DejaCode - - The process consists in load and iterate on every package distributions, - collect data and then acsk to save. """ - local_packages = get_local_packages(directory=dest_dir) - local_repo = get_local_repo(directory=dest_dir) - - remote_repo = get_remote_repo() - def get_other_dists(_package, _dist): """ - Return a list of all the dists from package that are not the `dist` object + Return a list of all the dists from `_package` that are not the `_dist` + object """ return [d for d in _package.get_distributions() if d != _dist] + selfhosted_repo = get_pypi_repo(index_url=ABOUT_PYPI_SIMPLE_URL) + local_packages = get_local_packages(directory=dest_dir) + packages_by_name = defaultdict(list) + for local_package in local_packages: + distributions = list(local_package.get_distributions()) + distribution = distributions[0] + packages_by_name[distribution.name].append(local_package) + for local_package in local_packages: for local_dist in local_package.get_distributions(): local_dist.load_about_data(dest_dir=dest_dir) local_dist.set_checksums(dest_dir=dest_dir) - if strip_classifiers and "classifiers" in local_dist.extra_data: - local_dist.extra_data.pop("classifiers", None) - local_dist.save_about_and_notice_files(dest_dir) - # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) @@ -2588,16 +2103,16 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - # try to get a latest version of the same package that is not our version + # try to get another version of the same package that is not our version other_local_packages = [ p - for p in local_repo.get_versions(local_package.name) + for p in packages_by_name[local_package.name] if p.version != local_package.version ] - latest_local_version = other_local_packages and other_local_packages[-1] - if latest_local_version: - latest_local_dists = list(latest_local_version.get_distributions()) + other_local_version = other_local_packages and other_local_packages[-1] + if other_local_version: + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2615,9 +2130,35 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - if include_remote: - # lets try to fetch remotely - local_dist.load_remote_about_data() + # lets try to fetch remotely + local_dist.load_remote_about_data() + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir) + continue + + # try to get a latest version of the same package that is not our version + other_remote_packages = [ + p + for p in selfhosted_repo.get_versions(local_package.name) + if p.version != local_package.version + ] + + latest_version = other_remote_packages and other_remote_packages[-1] + if latest_version: + latest_dists = list(latest_version.get_distributions()) + for remote_dist in latest_dists: + remote_dist.load_remote_about_data() + if not remote_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(remote_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): @@ -2625,33 +2166,6 @@ def get_other_dists(_package, _dist): local_dist.fetch_license_files(dest_dir=dest_dir) continue - # try to get a latest version of the same package that is not our version - other_remote_packages = [ - p - for p in remote_repo.get_versions(local_package.name) - if p.version != local_package.version - ] - - latest_version = other_remote_packages and other_remote_packages[-1] - if latest_version: - latest_dists = list(latest_version.get_distributions()) - for remote_dist in latest_dists: - remote_dist.load_remote_about_data() - if not remote_dist.has_key_metadata(): - # there is not much value to get other data if we are missing the key ones - continue - else: - local_dist.update_from_other_dist(remote_dist) - # if has key data we may look to improve later, but we can move on - if local_dist.has_key_metadata(): - break - - # if has key data we may look to improve later, but we can move on - if local_dist.has_key_metadata(): - local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) - continue - # try to get data from pkginfo (no license though) local_dist.load_pkginfo_data(dest_dir=dest_dir) @@ -2661,15 +2175,12 @@ def get_other_dists(_package, _dist): lic_errs = local_dist.fetch_license_files(dest_dir) - # TODO: try to get data from dejacode - if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") if lic_errs: lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") - ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2680,9 +2191,9 @@ def get_other_dists(_package, _dist): def call(args, verbose=TRACE): """ Call args in a subprocess and display output on the fly if ``trace`` is True. - Return or raise stdout, stderr, returncode + Return a tuple of (returncode, stdout, stderr) """ - if TRACE: + if TRACE_DEEP: print("Calling:", " ".join(args)) with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" @@ -2700,312 +2211,78 @@ def call(args, verbose=TRACE): stdout, stderr = process.communicate() if not stdout.strip(): stdout = "\n".join(stdouts) - - returncode = process.returncode - - if returncode == 0: - return returncode, stdout, stderr - else: - raise Exception(returncode, stdout, stderr) - - -def add_or_upgrade_built_wheels( - name, - version=None, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, - build_remotely=False, - with_deps=False, - verbose=TRACE, - remote_build_log_file=None, -): - """ - Add or update package `name` and `version` as a binary wheel saved in - `dest_dir`. Use the latest version if `version` is None. Return the a list - of the collected, fetched or built wheel file names or an empty list. - - Use the provided lists of `python_versions` (e.g. "36", "39") and - `operating_systems` (e.g. linux, windows or macos) to decide which specific - wheel to fetch or build. - - Include wheels for all dependencies if `with_deps` is True. - Build remotely is `build_remotely` is True. - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - assert name, "Name is required" - ver = version and f"=={version}" or "" - print(f"\nAdding wheels for package: {name}{ver}") - - if verbose: - print("python_versions:", python_versions) - print("operating_systems:", operating_systems) - - wheel_filenames = [] - # a mapping of {req specifier: {mapping build_wheels kwargs}} - wheels_to_build = {} - for python_version, operating_system in itertools.product(python_versions, operating_systems): - print( - f" Adding wheels for package: {name}{ver} on {python_version,} and {operating_system}" - ) - environment = Environment.from_pyver_and_os(python_version, operating_system) - - # Check if requested wheel already exists locally for this version - local_repo = get_local_repo(directory=dest_dir) - local_package = local_repo.get_package(name=name, version=version) - - has_local_wheel = False - if version and local_package: - for wheel in local_package.get_supported_wheels(environment): - has_local_wheel = True - wheel_filenames.append(wheel.filename) - break - if has_local_wheel: - print(f" local wheel exists: {wheel.filename}") - continue - - if not version: - pypi_package = get_pypi_repo().get_latest_version(name) - version = pypi_package.version - - # Check if requested wheel already exists remotely or in Pypi for this version - wheel_filename = fetch_package_wheel( - name=name, version=version, environment=environment, dest_dir=dest_dir - ) - if verbose: - print(" fetching package wheel:", wheel_filename) - if wheel_filename: - wheel_filenames.append(wheel_filename) - - # the wheel is not available locally, remotely or in Pypi - # we need to build binary from sources - requirements_specifier = f"{name}=={version}" - to_build = wheels_to_build.get(requirements_specifier) - if to_build: - to_build["python_versions"].append(python_version) - to_build["operating_systems"].append(operating_system) - else: - wheels_to_build[requirements_specifier] = dict( - requirements_specifier=requirements_specifier, - python_versions=[python_version], - operating_systems=[operating_system], - dest_dir=dest_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - for build_wheels_kwargs in wheels_to_build.values(): - bwheel_filenames = build_wheels(**build_wheels_kwargs) - wheel_filenames.extend(bwheel_filenames) - - return sorted(set(wheel_filenames)) - - -def build_wheels( - requirements_specifier, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - dest_dir=THIRDPARTY_DIR, - build_remotely=False, - with_deps=False, - verbose=False, - remote_build_log_file=None, -): - """ - Given a pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) for all - `python_versions` and `operating_systems` combinations and save them - back in `dest_dir` and return a list of built wheel file names. - - Include wheels for all dependencies if `with_deps` is True. - - First try to build locally to process pure Python wheels, and fall back to - build remotey on all requested Pythons and operating systems. - - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - all_pure, builds = build_wheels_locally_if_pure_python( - requirements_specifier=requirements_specifier, - with_deps=with_deps, - verbose=verbose, - dest_dir=dest_dir, - ) - for local_build in builds: - print(f"Built wheel: {local_build}") - - if all_pure: - return builds - - if build_remotely: - remote_builds = build_wheels_remotely_on_multiple_platforms( - requirements_specifier=requirements_specifier, - with_deps=with_deps, - python_versions=python_versions, - operating_systems=operating_systems, - verbose=verbose, - dest_dir=dest_dir, - remote_build_log_file=remote_build_log_file, - ) - builds.extend(remote_builds) - - return builds - - -def build_wheels_remotely_on_multiple_platforms( - requirements_specifier, - with_deps=False, - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, - verbose=False, - dest_dir=THIRDPARTY_DIR, - remote_build_log_file=None, -): - """ - Given pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) including wheels for - all dependencies for all `python_versions` and `operating_systems` - combinations and save them back in `dest_dir` and return a list of built - wheel file names. - - Do not wait for build completion and log to ``remote_build_log_file`` - file path if provided. - """ - check_romp_is_configured() - pyos_options = get_romp_pyos_options(python_versions, operating_systems) - deps = "" if with_deps else "--no-deps" - verbose = "--verbose" if verbose else "" - - if remote_build_log_file: - # zero seconds, no wait, log to file instead - wait_build_for = "0" - else: - wait_build_for = DEFAULT_ROMP_BUILD_WAIT - - romp_args = [ - "romp", - "--interpreter", - "cpython", - "--architecture", - "x86_64", - "--check-period", - wait_build_for, # in seconds - ] - - if remote_build_log_file: - romp_args += ["--build-log-file", remote_build_log_file] - - romp_args += pyos_options + [ - "--artifact-paths", - "*.whl", - "--artifact", - "artifacts.tar.gz", - "--command", - f"python -m pip {verbose} install --user --upgrade pip setuptools wheel; " - f"python -m pip {verbose} wheel {deps} {requirements_specifier}", - ] - - if verbose: - romp_args.append("--verbose") - - print(f"Building wheels for: {requirements_specifier}") - print(f"Using command:", " ".join(romp_args)) - call(romp_args) - wheel_filenames = [] - if not remote_build_log_file: - wheel_filenames = extract_tar("artifacts.tar.gz", dest_dir) - for wfn in wheel_filenames: - print(f" built wheel: {wfn}") - return wheel_filenames + return process.returncode, stdout, stderr -def fetch_remotely_built_wheels( - remote_build_log_file, +def download_wheels_with_pip( + requirements_specifiers=tuple(), + requirements_files=tuple(), + environment=None, dest_dir=THIRDPARTY_DIR, - no_wait=False, - verbose=False, + index_url=PYPI_SIMPLE_URL, + links_url=ABOUT_LINKS_URL, ): """ - Given a ``remote_build_log_file`` file path with a JSON lines log of a - remote build, fetch the built wheels and move them to ``dest_dir``. Return a - list of built wheel file names. - """ - wait = "0" if no_wait else DEFAULT_ROMP_BUILD_WAIT # in seconds - - romp_args = [ - "romp-fetch", - "--build-log-file", - remote_build_log_file, - "--check-period", - wait, + Fetch binary wheel(s) using pip for the ``envt`` Environment given a list of + pip ``requirements_files`` and a list of ``requirements_specifiers`` string + (such as package names or as name==version). + Return a tuple of (list of downloaded files, error string). + Do NOT fail on errors, but return an error message on failure. + """ + + cli_args = [ + "pip", + "download", + "--only-binary", + ":all:", + "--dest", + dest_dir, + "--index-url", + index_url, + "--find-links", + links_url, + "--no-color", + "--progress-bar", + "off", + "--no-deps", + "--no-build-isolation", + "--verbose", + # "--verbose", ] - if verbose: - romp_args.append("--verbose") - - print(f"Fetching built wheels from log file: {remote_build_log_file}") - print(f"Using command:", " ".join(romp_args)) - - call(romp_args, verbose=verbose) - - wheel_filenames = [] - - for art in os.listdir(os.getcwd()): - if not art.endswith("artifacts.tar.gz") or not os.path.getsize(art): - continue - - print(f" Processing artifact archive: {art}") - wheel_fns = extract_tar(art, dest_dir) - for wfn in wheel_fns: - print(f" Retrieved built wheel: {wfn}") - wheel_filenames.extend(wheel_fns) - return wheel_filenames - - -def get_romp_pyos_options( - python_versions=PYTHON_VERSIONS, - operating_systems=PLATFORMS_BY_OS, -): - """ - Return a list of CLI options for romp - For example: - >>> expected = ['--version', '3.6', '--version', '3.7', '--version', '3.8', - ... '--version', '3.9', '--version', '3.10', '--platform', 'linux', - ... '--platform', 'macos', '--platform', 'windows'] - >>> assert get_romp_pyos_options() == expected - """ - python_dot_versions = [get_python_dot_version(pv) for pv in sorted(set(python_versions))] - pyos_options = list( - itertools.chain.from_iterable(("--version", ver) for ver in python_dot_versions) - ) + if environment: + eopts = environment.get_pip_cli_options() + cli_args.extend(eopts) + else: + print("WARNING: no download environment provided.") - pyos_options += list( - itertools.chain.from_iterable( - ("--platform", plat) for plat in sorted(set(operating_systems)) - ) - ) + cli_args.extend(requirements_specifiers) + for req_file in requirements_files: + cli_args.extend(["--requirement", req_file]) - return pyos_options + if TRACE: + print(f"Downloading wheels using command:", " ".join(cli_args)) + existing = set(os.listdir(dest_dir)) + error = False + try: + returncode, _stdout, stderr = call(cli_args, verbose=True) + if returncode != 0: + error = stderr + except Exception as e: + error = str(e) -def check_romp_is_configured(): - # these environment variable must be set before - has_envt = ( - os.environ.get("ROMP_BUILD_REQUEST_URL") - and os.environ.get("ROMP_DEFINITION_ID") - and os.environ.get("ROMP_PERSONAL_ACCESS_TOKEN") - and os.environ.get("ROMP_USERNAME") - ) + if error: + print() + print("###########################################################################") + print("##################### Failed to fetch all wheels ##########################") + print("###########################################################################") + print(error) + print() + print("###########################################################################") - if not has_envt: - raise Exception( - "ROMP_BUILD_REQUEST_URL, ROMP_DEFINITION_ID, " - "ROMP_PERSONAL_ACCESS_TOKEN and ROMP_USERNAME " - "are required enironment variables." - ) + downloaded = existing ^ set(os.listdir(dest_dir)) + return sorted(downloaded), error def build_wheels_locally_if_pure_python( @@ -3034,9 +2311,9 @@ def build_wheels_locally_if_pure_python( "--wheel-dir", wheel_dir, ] - + deps - + verbose - + [requirements_specifier] + +deps + +verbose + +[requirements_specifier] ) print(f"Building local wheels for: {requirements_specifier}") @@ -3064,95 +2341,6 @@ def build_wheels_locally_if_pure_python( return all_pure, pure_built -# TODO: Use me -def optimize_wheel(wheel_filename, dest_dir=THIRDPARTY_DIR): - """ - Optimize a wheel named `wheel_filename` in `dest_dir` such as renaming its - tags for PyPI compatibility and making it smaller if possible. Return the - name of the new wheel if renamed or the existing new name otherwise. - """ - if is_pure_wheel(wheel_filename): - print(f"Pure wheel: {wheel_filename}, nothing to do.") - return wheel_filename - - original_wheel_loc = os.path.join(dest_dir, wheel_filename) - wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-") - awargs = ["auditwheel", "addtag", "--wheel-dir", wheel_dir, original_wheel_loc] - call(awargs) - - audited = os.listdir(wheel_dir) - if not audited: - # cannot optimize wheel - return wheel_filename - - assert len(audited) == 1 - new_wheel_name = audited[0] - - new_wheel_loc = os.path.join(wheel_dir, new_wheel_name) - - # this needs to go now - os.remove(original_wheel_loc) - - if new_wheel_name == wheel_filename: - os.rename(new_wheel_loc, original_wheel_loc) - return wheel_filename - - new_wheel = Wheel.from_filename(new_wheel_name) - non_pypi_plats = utils_pypi_supported_tags.validate_platforms_for_pypi(new_wheel.platforms) - new_wheel.platforms = [p for p in new_wheel.platforms if p not in non_pypi_plats] - if not new_wheel.platforms: - print(f"Cannot make wheel PyPI compatible: {original_wheel_loc}") - os.rename(new_wheel_loc, original_wheel_loc) - return wheel_filename - - new_wheel_cleaned_filename = new_wheel.to_filename() - new_wheel_cleaned_loc = os.path.join(dest_dir, new_wheel_cleaned_filename) - os.rename(new_wheel_loc, new_wheel_cleaned_loc) - return new_wheel_cleaned_filename - - -def extract_tar( - location, - dest_dir=THIRDPARTY_DIR, -): - """ - Extract a tar archive at `location` in the `dest_dir` directory. Return a - list of extracted locations (either directories or files). - """ - with open(location, "rb") as fi: - with tarfile.open(fileobj=fi) as tar: - members = list(tar.getmembers()) - tar.extractall(dest_dir, members=members) - - return [os.path.basename(ti.name) for ti in members if ti.type == tarfile.REGTYPE] - - -def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR): - """ - Fetch the binary wheel for package `name` and `version` and save in - `dest_dir`. Use the provided `environment` Environment to determine which - specific wheel to fetch. - - Return the fetched wheel file name on success or None if it was not fetched. - Trying fetching from our own remote repo, then from PyPI. - """ - wheel_filename = None - remote_package = get_remote_package(name=name, version=version) - if TRACE: - print(" remote_package:", remote_package) - if remote_package: - wheel_filename = remote_package.fetch_wheel(environment=environment, dest_dir=dest_dir) - if wheel_filename: - return wheel_filename - - pypi_package = get_pypi_package(name=name, version=version) - if TRACE: - print(" pypi_package:", pypi_package) - if pypi_package: - wheel_filename = pypi_package.fetch_wheel(environment=environment, dest_dir=dest_dir) - return wheel_filename - - def check_about(dest_dir=THIRDPARTY_DIR): try: subprocess.check_output(f"about check {dest_dir}".split()) @@ -3195,6 +2383,9 @@ def find_problems( def compute_normalized_license_expression(declared_licenses): + """ + Return a normalized license expression or None. + """ if not declared_licenses: return try: From 931f610aa8fd93aac6178b1f0beb3d55d4119ba0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 5 Mar 2022 07:42:35 +0100 Subject: [PATCH 38/62] Cleanup whitespaces Signed-off-by: Philippe Ombredanne --- configure | 5 ++--- configure.bat | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/configure b/configure index c1d36aa..b2d64c4 100755 --- a/configure +++ b/configure @@ -55,7 +55,7 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin # Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then - PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty " + PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @@ -87,7 +87,7 @@ main() { done PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" - + find_python create_virtualenv "$VIRTUALENV_DIR" install_packages "$CFG_REQUIREMENTS" @@ -197,7 +197,6 @@ clean() { } - main set +e diff --git a/configure.bat b/configure.bat index 961e0d9..2ae4727 100644 --- a/configure.bat +++ b/configure.bat @@ -209,4 +209,4 @@ for %%F in (%CLEANABLE%) do ( rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 ) -exit /b 0 \ No newline at end of file +exit /b 0 From 6e43a7a2a98322fc3da3ed61826757481b831c50 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 8 Mar 2022 16:17:33 -0800 Subject: [PATCH 39/62] Add usage instructions to README.rst Signed-off-by: Jono Yang --- README.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4173689..26bcdbc 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,32 @@ our existing ones as well. Usage ===== -Usage instructions can be found in ``docs/skeleton-usage.rst``. + +A brand new project +------------------- +.. code-block:: bash + + git init my-new-repo + cd my-new-repo + git pull git@github.com:nexB/skeleton + + # Create the new repo on GitHub, then update your remote + git remote set-url origin git@github.com:nexB/your-new-repo.git + +From here, you can make the appropriate changes to the files for your specific project. + +Update an existing project +--------------------------- +.. code-block:: bash + + cd my-existing-project + git remote add skeleton git@github.com:nexB/skeleton + git fetch skeleton + git merge skeleton/main --allow-unrelated-histories + +This is also the workflow to use when updating the skeleton files in any given repository. + +More usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= From b272e3b7c7e47a3143e0886ebc9e88b12c1c6eab Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 14:08:43 +0100 Subject: [PATCH 40/62] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_thirdparty.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index e303053..829cf8c 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -330,6 +330,7 @@ def get_package_versions( except RemoteNotFetchedException as e: print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + ################################################################################ # # Core models @@ -1617,6 +1618,7 @@ def tags(self): ) ) + ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1801,6 +1803,7 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): _LINKS_REPO[url] = cls(url=url) return _LINKS_REPO[url] + ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1834,6 +1837,7 @@ def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP): except RemoteNotFetchedException as e: print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1994,6 +1998,7 @@ def fetch_and_save_path_or_url( fo.write(content) return content + ################################################################################ # Requirements processing ################################################################################ @@ -2031,6 +2036,7 @@ def get_required_packages( print(" get_required_packages: name:", name, "version:", version) yield repo.get_package(name, version) + ################################################################################ # Functions to update or fetch ABOUT and license files ################################################################################ @@ -2181,6 +2187,7 @@ def get_other_dists(_package, _dist): lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") + ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2311,9 +2318,9 @@ def build_wheels_locally_if_pure_python( "--wheel-dir", wheel_dir, ] - +deps - +verbose - +[requirements_specifier] + + deps + + verbose + + [requirements_specifier] ) print(f"Building local wheels for: {requirements_specifier}") From 1e4d3bce4626494bb1392a063360e236caf77294 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 16:56:19 +0100 Subject: [PATCH 41/62] Reorg setup sections This is now organized with more important data first. Signed-off-by: Philippe Ombredanne --- setup.cfg | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/setup.cfg b/setup.cfg index 81f762a..d8a7941 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,18 +1,15 @@ [metadata] -license_files = - apache-2.0.LICENSE - NOTICE - AUTHORS.rst - CHANGELOG.rst name = skeleton -author = nexB. Inc. and others -author_email = info@aboutcode.org license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 description = skeleton long_description = file:README.rst url = https://github.com/nexB/skeleton + +author = nexB. Inc. and others +author_email = info@aboutcode.org + classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Developers @@ -20,27 +17,42 @@ classifiers = Programming Language :: Python :: 3 :: Only Topic :: Software Development Topic :: Utilities + keywords = utilities +license_files = + apache-2.0.LICENSE + NOTICE + AUTHORS.rst + CHANGELOG.rst + [options] -package_dir= +package_dir = =src -packages=find: +packages = find: include_package_data = true zip_safe = false -install_requires = + setup_requires = setuptools_scm[toml] >= 4 +python_requires = >=3.6.* + +install_requires = + + [options.packages.find] -where=src +where = src + [options.extras_require] testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 + aboutcode-toolkit >= 6.0.0 black -docs= - Sphinx>=3.3.1 - sphinx-rtd-theme>=0.5.0 - doc8>=0.8.1 + +docs = + Sphinx >= 3.3.1 + sphinx-rtd-theme >= 0.5.0 + doc8 >= 0.8.1 From 03d4799ac44a4def7dba1eb3a0ef3a280c663e43 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 20:39:36 +0100 Subject: [PATCH 42/62] Do not depend on click. Use argparse. These boostrap scripts cannot depend on click. Signed-off-by: Philippe Ombredanne --- docs/skeleton-usage.rst | 16 +++--- etc/scripts/gen_requirements.py | 59 +++++++++++++--------- etc/scripts/gen_requirements_dev.py | 78 ++++++++++++++++------------- 3 files changed, 86 insertions(+), 67 deletions(-) diff --git a/docs/skeleton-usage.rst b/docs/skeleton-usage.rst index 7d16259..113bc71 100644 --- a/docs/skeleton-usage.rst +++ b/docs/skeleton-usage.rst @@ -49,7 +49,7 @@ customizing the skeleton files to your project: .. code-block:: bash - ./configure --init + ./configure This will initialize the virtual environment for the project, pull in the dependencies from PyPI and add them to the virtual environment. @@ -77,7 +77,7 @@ Replace \ with the version number of the Python being used, for exampl To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash - ./configure --init --dev + ./configure --dev python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` @@ -88,10 +88,11 @@ Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site- .\configure --init --dev python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ + Collecting and generating ABOUT files for dependencies ------------------------------------------------------ -Ensure that the dependencies used by ``etc/scripts/bootstrap.py`` are installed: +Ensure that the dependencies used by ``etc/scripts/fetch_thirdparty.py`` are installed: .. code-block:: bash @@ -102,7 +103,7 @@ dependencies as wheels and generate ABOUT files for them: .. code-block:: bash - python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps + python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt There may be issues with the generated ABOUT files, which will have to be corrected. You can check to see if your corrections are valid by running: @@ -122,8 +123,8 @@ Usage after project initialization Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated and the project dependencies and their ABOUT files have been uploaded to -thirdparty.aboutcode.org/pypi, you can configure the project without using the -``--init`` option. +thirdparty.aboutcode.org/pypi, you can configure the project as needed, typically +when you update dependencies or use a new checkout. If the virtual env for the project becomes polluted, or you would like to remove it, use the ``--clean`` option: @@ -146,12 +147,11 @@ update the dependencies in ``setup.cfg``, then run: .. code-block:: bash ./configure --clean # Remove existing virtual environment - ./configure --init # Create project virtual environment, pull in new dependencies source venv/bin/activate # Ensure virtual environment is activated python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py - python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps # Collect dependency wheels and their ABOUT files + python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt # Collect dependency wheels and their ABOUT files Ensure that the generated ABOUT files are valid, then take the dependency wheels and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 6f17a75..07e26f7 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -8,37 +8,48 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click +import argparse +import pathlib + import utils_requirements +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" -@click.command() -@click.option( - "-s", - "--site-packages-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, - metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', -) -@click.option( - "-r", - "--requirements-file", - type=click.Path(path_type=str, dir_okay=False), - metavar="FILE", - default="requirements.txt", - show_default=True, - help="Path to the requirements file to update or create.", -) -@click.help_option("-h", "--help") -def gen_requirements(site_packages_dir, requirements_file): - """ + +def gen_requirements(): + description = """ Create or replace the `--requirements-file` file FILE requirements file with all locally installed Python packages.all Python packages found installed in `--site-packages-dir` """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + dest="site_packages_dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + ) + parser.add_argument( + "-r", + "--requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements.txt", + help="Path to the requirements file to update or create.", + ) + + args = parser.parse_args() + utils_requirements.lock_requirements( - requirements_file=requirements_file, - site_packages_dir=site_packages_dir, + site_packages_dir=args.site_packages_dir, + requirements_file=args.requirements_file, ) diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index ef80455..12cc06d 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -8,51 +8,59 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click +import argparse +import pathlib + import utils_requirements +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" -@click.command() -@click.option( - "-s", - "--site-packages-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, - metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', -) -@click.option( - "-d", - "--dev-requirements-file", - type=click.Path(path_type=str, dir_okay=False), - metavar="FILE", - default="requirements-dev.txt", - show_default=True, - help="Path to the dev requirements file to update or create.", -) -@click.option( - "-r", - "--main-requirements-file", - type=click.Path(path_type=str, dir_okay=False), - default="requirements.txt", - metavar="FILE", - show_default=True, - help="Path to the main requirements file. Its requirements will be excluded " - "from the generated dev requirements.", -) -@click.help_option("-h", "--help") -def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirements_file): - """ + +def gen_dev_requirements(): + description = """ Create or overwrite the `--dev-requirements-file` pip requirements FILE with all Python packages found installed in `--site-packages-dir`. Exclude package names also listed in the --main-requirements-file pip requirements FILE (that are assume to the production requirements and therefore to always be present in addition to the development requirements). """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + ) + parser.add_argument( + "-d", + "--dev-requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements-dev.txt", + help="Path to the dev requirements file to update or create.", + ) + parser.add_argument( + "-r", + "--main-requirements-file", + type=pathlib.Path, + default="requirements.txt", + metavar="FILE", + help="Path to the main requirements file. Its requirements will be excluded " + "from the generated dev requirements.", + ) + args = parser.parse_args() + utils_requirements.lock_dev_requirements( - dev_requirements_file=dev_requirements_file, - main_requirements_file=main_requirements_file, - site_packages_dir=site_packages_dir, + dev_requirements_file=args.dev_requirements_file, + main_requirements_file=args.main_requirements_file, + site_packages_dir=args.site_packages_dir, ) From f0d5a2979c9e04c7f77c5cf76aea5c936ee31ac3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 20:47:06 +0100 Subject: [PATCH 43/62] Correct configure scripts These were no longer working Signed-off-by: Philippe Ombredanne --- configure | 53 +++++++++++++++++++++++---------------------------- configure.bat | 5 +---- 2 files changed, 25 insertions(+), 33 deletions(-) diff --git a/configure b/configure index b2d64c4..93a836b 100755 --- a/configure +++ b/configure @@ -67,34 +67,6 @@ if [[ "$CFG_QUIET" == "" ]]; then fi -################################ -# Main command line entry point -main() { - CFG_REQUIREMENTS=$REQUIREMENTS - NO_INDEX="--no-index" - - # We are using getopts to parse option arguments that start with "-" - while getopts :-: optchar; do - case "${optchar}" in - -) - case "${OPTARG}" in - help ) cli_help;; - clean ) find_python && clean;; - dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - init ) NO_INDEX="";; - esac;; - esac - done - - PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" - - find_python - create_virtualenv "$VIRTUALENV_DIR" - install_packages "$CFG_REQUIREMENTS" - . "$CFG_BIN_DIR/activate" -} - - ################################ # Find a proper Python to run # Use environment variables or a file if available. @@ -197,6 +169,29 @@ clean() { } -main +################################ +# Main command line entry point +CFG_REQUIREMENTS=$REQUIREMENTS + +# We are using getopts to parse option arguments that start with "-" +while getopts :-: optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help ) cli_help;; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + init ) ;; + esac;; + esac +done + +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" + +find_python +create_virtualenv "$VIRTUALENV_DIR" +install_packages "$CFG_REQUIREMENTS" +. "$CFG_BIN_DIR/activate" + set +e diff --git a/configure.bat b/configure.bat index 2ae4727..7001514 100644 --- a/configure.bat +++ b/configure.bat @@ -77,14 +77,11 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) - if "%1" EQU "--init" ( - set "NO_INDEX= " - ) shift goto again ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" @rem ################################ From 6ed9983e882b195b3093434f70fd0d0f01d8399f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 9 Mar 2022 20:50:04 +0100 Subject: [PATCH 44/62] Remove remnants of configure --init This does not exists anymore Signed-off-by: Philippe Ombredanne --- configure | 5 ----- configure.bat | 4 ---- docs/skeleton-usage.rst | 3 ++- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/configure b/configure index 93a836b..8c5f4ab 100755 --- a/configure +++ b/configure @@ -137,14 +137,10 @@ cli_help() { echo " usage: ./configure [options]" echo echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. echo echo The options are: echo " --clean: clean built and installed files and exit." echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." echo " --help: display this help message and exit." echo echo By default, the python interpreter version found in the path is used. @@ -181,7 +177,6 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - init ) ;; esac;; esac done diff --git a/configure.bat b/configure.bat index 7001514..e38b5fb 100644 --- a/configure.bat +++ b/configure.bat @@ -180,14 +180,10 @@ exit /b 0 echo " usage: configure [options]" echo " " echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. echo " " echo The options are: echo " --clean: clean built and installed files and exit." echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." echo " --help: display this help message and exit." echo " " echo By default, the python interpreter version found in the path is used. diff --git a/docs/skeleton-usage.rst b/docs/skeleton-usage.rst index 113bc71..ad9b9ff 100644 --- a/docs/skeleton-usage.rst +++ b/docs/skeleton-usage.rst @@ -54,6 +54,7 @@ customizing the skeleton files to your project: This will initialize the virtual environment for the project, pull in the dependencies from PyPI and add them to the virtual environment. + Generating requirements.txt and requirements-dev.txt ---------------------------------------------------- @@ -85,7 +86,7 @@ Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site- .. code-block:: bash python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ - .\configure --init --dev + .\configure --dev python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ From bf6bbaac67cb8fa31d205b2d76e538a3bed8780f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 11 Mar 2022 07:52:07 +0100 Subject: [PATCH 45/62] Pytyon 3.6 is not available on Windows 2022 Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 089abe9..6ca19c4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macos-11 - python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_versions: ['3.7', '3.8', '3.9', '3.10'] test_suites: all: venv/bin/pytest -n 2 -vvs From 4ab834f15860a22bcbca2a5ea567ce5f39c7c345 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 11 Mar 2022 09:42:46 +0100 Subject: [PATCH 46/62] Add long_description_content_type Twine and PyPI prefer having it. Signed-off-by: Philippe Ombredanne --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index d8a7941..12d6654 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,7 @@ license = Apache-2.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 description = skeleton long_description = file:README.rst +long_description_content_type = text/x-rst url = https://github.com/nexB/skeleton author = nexB. Inc. and others From 4ef463fdbbcc1c108307b07e26b4a231d2229799 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 14 Mar 2022 11:12:54 +0100 Subject: [PATCH 47/62] Run fewer Azure jobs This new configuration means that all the Python versions are tested in a single CI job. This allows doing fewer checkouts and improves CI throughput overall. Signed-off-by: Philippe Ombredanne --- etc/ci/azure-posix.yml | 31 ++++++++++++++----------------- etc/ci/azure-win.yml | 30 +++++++++++++++--------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 7a9acff..9fdc7f1 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -13,10 +13,8 @@ jobs: strategy: matrix: - ${{ each pyver in parameters.python_versions }}: ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} + ${{ tsuite.key }}: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} @@ -24,19 +22,18 @@ jobs: - checkout: self fetchDepth: 10 - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' - - script: | - python --version - python3 --version - python$(python_version) --version - echo "python$(python_version)" > PYTHON_EXECUTABLE - ./configure --dev - displayName: 'Run Configure' + - script: | + python${{ pyver }} --version + echo "python${{ pyver }}" > PYTHON_EXECUTABLE + ./configure --clean && ./configure --dev + displayName: '${{ pyver }} - Configure' - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index 03d8927..26b4111 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -13,27 +13,27 @@ jobs: strategy: matrix: - ${{ each pyver in parameters.python_versions }}: ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} + ${{ tsuite.key }}: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} + steps: - checkout: self fetchDepth: 10 - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' - - script: | - python --version - echo | set /p=python> PYTHON_EXECUTABLE - configure --dev - displayName: 'Run Configure' + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --clean && configure --dev + displayName: '${{ pyver }} - Configure' - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' From e9210529fbe09a498abf85d27154110a34728fcb Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 22 Mar 2022 21:13:44 +0530 Subject: [PATCH 48/62] Add RTD css templates Adds changes to conf.py and html template theme_overrides.css created by @johnmhoran Signed-off-by: Ayan Sinha Mahapatra --- docs/source/_static/theme_overrides.css | 353 ++++++++++++++++++++++++ docs/source/conf.py | 21 ++ 2 files changed, 374 insertions(+) create mode 100644 docs/source/_static/theme_overrides.css diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 0000000..9662d63 --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,353 @@ +body { + color: #000000; +} + +p { + margin-bottom: 10px; +} + +.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { + margin-bottom: 10px; +} + +.custom_header_01 { + color: #cc0000; + font-size: 22px; + font-weight: bold; + line-height: 50px; +} + +h1, h2, h3, h4, h5, h6 { + margin-bottom: 20px; + margin-top: 20px; +} + +h5 { + font-size: 18px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +h6 { + font-size: 15px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +/* custom admonitions */ +/* success */ +.custom-admonition-success .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-success.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* important */ +.custom-admonition-important .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #000000; +} +div.custom-admonition-important.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* caution */ +.custom-admonition-caution .admonition-title { + color: #000000; + background: #ffff99; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #e8e8e8; +} +div.custom-admonition-caution.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* note */ +.custom-admonition-note .admonition-title { + color: #ffffff; + background: #006bb3; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-note.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* todo */ +.custom-admonition-todo .admonition-title { + color: #000000; + background: #cce6ff; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #99ccff; +} +div.custom-admonition-todo.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #99ccff; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* examples */ +.custom-admonition-examples .admonition-title { + color: #000000; + background: #ffe6cc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #d8d8d8; +} +div.custom-admonition-examples.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +.wy-nav-content { + max-width: 100%; + padding-right: 100px; + padding-left: 100px; + background-color: #f2f2f2; +} + +div.rst-content { + background-color: #ffffff; + border: solid 1px #e5e5e5; + padding: 20px 40px 20px 40px; +} + +.rst-content .guilabel { + border: 1px solid #ffff99; + background: #ffff99; + font-size: 100%; + font-weight: normal; + border-radius: 4px; + padding: 2px 0px; + margin: auto 2px; + vertical-align: middle; +} + +.rst-content kbd { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + border: solid 1px #d8d8d8; + background-color: #f5f5f5; + padding: 0px 3px; + border-radius: 3px; +} + +.wy-nav-content-wrap a { + color: #0066cc; + text-decoration: none; +} +.wy-nav-content-wrap a:hover { + color: #0099cc; + text-decoration: underline; +} + +.wy-nav-top a { + color: #ffffff; +} + +/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ +.wy-table-responsive table td { + white-space: normal !important; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 5px 10px 5px 10px; +} +.rst-content table.docutils td p, +.rst-content table.docutils th p { + font-size: 14px; + margin-bottom: 0px; +} +.rst-content table.docutils td p cite, +.rst-content table.docutils th p cite { + font-size: 14px; + background-color: transparent; +} + +.colwidths-given th { + border: solid 1px #d8d8d8 !important; +} +.colwidths-given td { + border: solid 1px #d8d8d8 !important; +} + +/*handles single-tick inline code*/ +.wy-body-for-nav cite { + color: #000000; + background-color: transparent; + font-style: normal; + font-family: "Courier New"; + font-size: 13px; + padding: 3px 3px 3px 3px; +} + +.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + font-size: 13px; + overflow: visible; + white-space: pre-wrap; + color: #000000; +} + +.rst-content pre.literal-block, .rst-content div[class^='highlight'] { + background-color: #f8f8f8; + border: solid 1px #e8e8e8; +} + +/* This enables inline code to wrap. */ +code, .rst-content tt, .rst-content code { + white-space: pre-wrap; + padding: 2px 3px 1px; + border-radius: 3px; + font-size: 13px; + background-color: #ffffff; +} + +/* use this added class for code blocks attached to bulleted list items */ +.highlight-top-margin { + margin-top: 20px !important; +} + +/* change color of inline code block */ +span.pre { + color: #e01e5a; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #000000; +} + +/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} + +/* add spacing between bullets for legibility */ +.rst-content .section ol li, .rst-content .section ul li { + margin-bottom: 5px; +} + +.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { + margin-top: 5px; +} + +/* but exclude the toctree bullets */ +.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { + margin-top: 0px; + margin-bottom: 0px; +} + +/* remove extra space at bottom of multine list-table cell */ +.rst-content .line-block { + margin-left: 0px; + margin-bottom: 0px; + line-height: 24px; +} + +/* fix extra vertical spacing in page toctree */ +.rst-content .toctree-wrapper ul li ul, article ul li ul { + margin-top: 0; + margin-bottom: 0; +} + +/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ +.reference.internal.toc-index { + color: #d9d9d9; +} + +.reference.internal.toc-index.current { + background-color: #ffffff; + color: #000000; + font-weight: bold; +} + +.toc-index-div { + border-top: solid 1px #000000; + margin-top: 10px; + padding-top: 5px; +} + +.indextable ul li { + font-size: 14px; + margin-bottom: 5px; +} + +/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ +.indextable.genindextable { + margin-bottom: 20px; +} + +div.genindex-jumpbox { + margin-bottom: 10px; +} + +/* rst image classes */ + +.clear-both { + clear: both; + } + +.float-left { + float: left; + margin-right: 20px; +} + +img { + border: solid 1px #e8e8e8; +} + +/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ +.img-title { + color: #000000; + /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ + line-height: 3.0; + font-style: italic; + font-weight: 600; +} + +.img-title-para { + color: #000000; + margin-top: 20px; + margin-bottom: 0px; + font-style: italic; + font-weight: 500; +} + +.red { + color: red; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 74b8649..778636e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -74,3 +74,24 @@ "github_version": "develop", # branch "conf_py_path": "/docs/source/", # path in the checkout to the docs root } + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +html_show_sphinx = True + +# Define CSS and HTML abbreviations used in .rst files. These are examples. +# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` +rst_prolog = """ +.. |psf| replace:: Python Software Foundation + +.. # define a hard line break for HTML +.. |br| raw:: html + +
+ +.. role:: red + +.. role:: img-title + +.. role:: img-title-para + +""" From bd2df2a9608fe02e5d725ab8d9b03a00fdb0ed7a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 15:12:18 +0530 Subject: [PATCH 49/62] Add GitHub action for doc build tests Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 37 ++++++++++++++++++++++++++++++++++ docs/source/skeleton/index.rst | 4 ++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docs-ci.yml diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 0000000..656d624 --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,37 @@ +name: CI Documentation + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-20.04 + + strategy: + max-parallel: 4 + matrix: + python-version: [3.7] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Give permission to run scripts + run: chmod +x ./docs/scripts/doc8_style_check.sh + + - name: Install Dependencies + run: pip install -e .[docs] + + - name: Check Sphinx Documentation build minimally + working-directory: ./docs + run: sphinx-build -E -W source build + + - name: Check for documentation style errors + working-directory: ./docs + run: ./scripts/doc8_style_check.sh + + diff --git a/docs/source/skeleton/index.rst b/docs/source/skeleton/index.rst index 7dfc6cb..f99cdec 100644 --- a/docs/source/skeleton/index.rst +++ b/docs/source/skeleton/index.rst @@ -2,14 +2,14 @@ # Rst docs - https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html # # 1. Place docs in folders under source for different sections -# 2. Link them by adding individual index files in each section +# 2. Link them by adding individual index files in each section # to the main index, and then files for each section to their # respective index files. # 3. Use `.. include` statements to include other .rst files # or part of them, or use hyperlinks to a section of the docs, # to get rid of repetition. # https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment -# +# # Note: Replace these guide/placeholder docs .. include:: ../../../README.rst From 3e2d801c69cc1c7523d1613bc9c3e3d805b85d3b Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 15:59:39 +0530 Subject: [PATCH 50/62] Fix conf.py to fix doc build Signed-off-by: Ayan Sinha Mahapatra --- docs/source/conf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 778636e..62bca04 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -65,9 +65,6 @@ master_doc = 'index' html_context = { - "css_files": [ - "_static/theme_overrides.css", # override wide tables in RTD theme - ], "display_github": True, "github_user": "nexB", "github_repo": "nexb-skeleton", @@ -75,6 +72,11 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } +html_css_files = [ + '_static/theme_overrides.css' + ] + + # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = True From eb578898a0f48ca04f5071dbbfb460de35eb5383 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 24 Mar 2022 20:21:44 +0530 Subject: [PATCH 51/62] Add docs option in configure Adds a --docs option to the configure script to also install requirements for the documentation builds. Signed-off-by: Ayan Sinha Mahapatra --- configure | 2 ++ configure.bat | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/configure b/configure index 8c5f4ab..715c99f 100755 --- a/configure +++ b/configure @@ -30,6 +30,7 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv VIRTUALENV_DIR=venv @@ -177,6 +178,7 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/configure.bat b/configure.bat index e38b5fb..487e78a 100644 --- a/configure.bat +++ b/configure.bat @@ -28,6 +28,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -77,6 +78,9 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) + if "%1" EQU "--docs" ( + set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" + ) shift goto again ) From 5556e71f0e3f780b4dd955e1f3b93395d345c36c Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 30 Mar 2022 15:28:39 +0530 Subject: [PATCH 52/62] Add documentation contribute page Adds documentation page on contributing to the docs, and also modifies directory structure to avoid having the skeleton directory in docs merged in projects. Signed-off-by: Ayan Sinha Mahapatra --- docs/source/contribute/contrib_doc.rst | 314 +++++++++++++++++++++++++ docs/source/index.rst | 3 +- docs/{ => source}/skeleton-usage.rst | 4 +- docs/source/skeleton/index.rst | 15 -- 4 files changed, 319 insertions(+), 17 deletions(-) create mode 100644 docs/source/contribute/contrib_doc.rst rename docs/{ => source}/skeleton-usage.rst (98%) delete mode 100644 docs/source/skeleton/index.rst diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst new file mode 100644 index 0000000..13882e1 --- /dev/null +++ b/docs/source/contribute/contrib_doc.rst @@ -0,0 +1,314 @@ +.. _contrib_doc_dev: + +Contributing to the Documentation +================================= + +.. _contrib_doc_setup_local: + +Setup Local Build +----------------- + +To get started, create or identify a working directory on your local machine. + +Open that directory and execute the following command in a terminal session:: + + git clone https://github.com/nexB/skeleton.git + +That will create an ``/skeleton`` directory in your working directory. +Now you can install the dependencies in a virtualenv:: + + cd skeleton + ./configure --docs + +.. note:: + + In case of windows, run ``configure --docs`` instead of this. + +Now, this will install the following prerequisites: + +- Sphinx +- sphinx_rtd_theme (the format theme used by ReadTheDocs) +- docs8 (style linter) + +These requirements are already present in setup.cfg and `./configure --docs` installs them. + +Now you can build the HTML documents locally:: + + source venv/bin/activate + cd docs + make html + +Assuming that your Sphinx installation was successful, Sphinx should build a local instance of the +documentation .html files:: + + open build/html/index.html + +.. note:: + + In case this command did not work, for example on Ubuntu 18.04 you may get a message like “Couldn’t + get a file descriptor referring to the console”, try: + + :: + + see build/html/index.html + +You now have a local build of the AboutCode documents. + +.. _contrib_doc_share_improvements: + +Share Document Improvements +--------------------------- + +Ensure that you have the latest files:: + + git pull + git status + +Before commiting changes run Continious Integration Scripts locally to run tests. Refer +:ref:`doc_ci` for instructions on the same. + +Follow standard git procedures to upload your new and modified files. The following commands are +examples:: + + git status + git add source/index.rst + git add source/how-to-scan.rst + git status + git commit -m "New how-to document that explains how to scan" + git status + git push + git status + +The Scancode-Toolkit webhook with ReadTheDocs should rebuild the documentation after your +Pull Request is Merged. + +Refer the `Pro Git Book `_ available online for Git tutorials +covering more complex topics on Branching, Merging, Rebasing etc. + +.. _doc_ci: + +Continuous Integration +---------------------- + +The documentations are checked on every new commit through Travis-CI, so that common errors are +avoided and documentation standards are enforced. Travis-CI presently checks for these 3 aspects +of the documentation : + +1. Successful Builds (By using ``sphinx-build``) +2. No Broken Links (By Using ``link-check``) +3. Linting Errors (By Using ``Doc8``) + +So run these scripts at your local system before creating a Pull Request:: + + cd docs + ./scripts/sphinx_build_link_check.sh + ./scripts/doc8_style_check.sh + +If you don't have permission to run the scripts, run:: + + chmod u+x ./scripts/doc8_style_check.sh + +.. _doc_style_docs8: + +Style Checks Using ``Doc8`` +--------------------------- + +How To Run Style Tests +^^^^^^^^^^^^^^^^^^^^^^ + +In the project root, run the following commands:: + + $ cd docs + $ ./scripts/doc8_style_check.sh + +A sample output is:: + + Scanning... + Validating... + docs/source/misc/licence_policy_plugin.rst:37: D002 Trailing whitespace + docs/source/misc/faq.rst:45: D003 Tabulation used for indentation + docs/source/misc/faq.rst:9: D001 Line too long + docs/source/misc/support.rst:6: D005 No newline at end of file + ======== + Total files scanned = 34 + Total files ignored = 0 + Total accumulated errors = 326 + Detailed error counts: + - CheckCarriageReturn = 0 + - CheckIndentationNoTab = 75 + - CheckMaxLineLength = 190 + - CheckNewlineEndOfFile = 13 + - CheckTrailingWhitespace = 47 + - CheckValidity = 1 + +Now fix the errors and run again till there isn't any style error in the documentation. + +What is Checked? +^^^^^^^^^^^^^^^^ + +PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. + +What is checked: + + - invalid rst format - D000 + - lines should not be longer than 100 characters - D001 + + - RST exception: line with no whitespace except in the beginning + - RST exception: lines with http or https URLs + - RST exception: literal blocks + - RST exception: rst target directives + + - no trailing whitespace - D002 + - no tabulation for indentation - D003 + - no carriage returns (use UNIX newlines) - D004 + - no newline at end of file - D005 + +.. _doc_interspinx: + +Interspinx +---------- + +ScanCode toolkit documentation uses `Intersphinx `_ +to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. + +To link sections in the same documentation, standart reST labels are used. Refer +`Cross-Referencing `_ for more information. + +For example:: + + .. _my-reference-label: + + Section to cross-reference + -------------------------- + + This is the text of the section. + + It refers to the section itself, see :ref:`my-reference-label`. + +Now, using Intersphinx, you can create these labels in one Sphinx Documentation and then referance +these labels from another Sphinx Documentation, hosted in different locations. + +You just have to add the following in the ``conf.py`` file for your Sphinx Documentation, where you +want to add the links:: + + extensions = [ + 'sphinx.ext.intersphinx' + ] + + intersphinx_mapping = {'aboutcode': ('https://aboutcode.readthedocs.io/en/latest/', None)} + +To show all Intersphinx links and their targets of an Intersphinx mapping file, run:: + + python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/en/latest/objects.inv + +.. WARNING:: + + ``python -msphinx.ext.intersphinx https://aboutcode.readthedocs.io/objects.inv`` will give + error. + +This enables you to create links to the ``aboutcode`` Documentation in your own Documentation, +where you modified the configuration file. Links can be added like this:: + + For more details refer :ref:`aboutcode:doc_style_guide`. + +You can also not use the ``aboutcode`` label assigned to all links from aboutcode.readthedocs.io, +if you don't have a label having the same name in your Sphinx Documentation. Example:: + + For more details refer :ref:`doc_style_guide`. + +If you have a label in your documentation which is also present in the documentation linked by +Intersphinx, and you link to that label, it will create a link to the local label. + +For more information, refer this tutorial named +`Using Intersphinx `_. + +.. _doc_style_conv: + +Style Conventions for the Documentaion +-------------------------------------- + +1. Headings + + (`Refer `_) + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s Style + Guide for documenting which you may follow: + + # with overline, for parts + + * with overline, for chapters + + =, for sections + + -, for subsections + + ^, for sub-subsections + + ", for paragraphs + +2. Heading Underlines + + Do not use underlines that are longer/shorter than the title headline itself. As in: + + :: + + Correct : + + Extra Style Checks + ------------------ + + Incorrect : + + Extra Style Checks + ------------------------ + +.. note:: + + Underlines shorter than the Title text generates Errors on sphinx-build. + + +3. Internal Links + + Using ``:ref:`` is advised over standard reStructuredText links to sections (like + ```Section title`_``) because it works across files, when section headings are changed, will + raise warnings if incorrect, and works for all builders that support cross-references. + However, external links are created by using the standard ```Section title`_`` method. + +4. Eliminate Redundancy + + If a section/file has to be repeated somewhere else, do not write the exact same section/file + twice. Use ``.. include: ../README.rst`` instead. Here, ``../`` refers to the documentation + root, so file location can be used accordingly. This enables us to link documents from other + upstream folders. + +5. Using ``:ref:`` only when necessary + + Use ``:ref:`` to create internal links only when needed, i.e. it is referenced somewhere. + Do not create references for all the sections and then only reference some of them, because + this created unnecessary references. This also generates ERROR in ``restructuredtext-lint``. + +6. Spelling + + You should check for spelling errors before you push changes. `Aspell `_ + is a GNU project Command Line tool you can use for this purpose. Download and install Aspell, + then execute ``aspell check `` for all the files changed. Be careful about not + changing commands or other stuff as Aspell gives prompts for a lot of them. Also delete the + temporary ``.bak`` files generated. Refer the `manual `_ for more + information on how to use. + +7. Notes and Warning Snippets + + Every ``Note`` and ``Warning`` sections are to be kept in ``rst_snippets/note_snippets/`` and + ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are + frequently used in multiple files. + +Converting from Markdown +------------------------ + +If you want to convert a ``.md`` file to a ``.rst`` file, this `tool `_ +does it pretty well. You'd still have to clean up and check for errors as this contains a lot of +bugs. But this is definitely better than converting everything by yourself. + +This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for +Sphinx/ReadTheDocs hosting. diff --git a/docs/source/index.rst b/docs/source/index.rst index 67fcf21..eb63717 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,7 +5,8 @@ Welcome to nexb-skeleton's documentation! :maxdepth: 2 :caption: Contents: - skeleton/index + skeleton-usage + contribute/contrib_doc Indices and tables ================== diff --git a/docs/skeleton-usage.rst b/docs/source/skeleton-usage.rst similarity index 98% rename from docs/skeleton-usage.rst rename to docs/source/skeleton-usage.rst index ad9b9ff..cde23dc 100644 --- a/docs/skeleton-usage.rst +++ b/docs/source/skeleton-usage.rst @@ -73,11 +73,13 @@ To generate requirements.txt: python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ -Replace \ with the version number of the Python being used, for example: ``venv/lib/python3.6/site-packages/`` +Replace \ with the version number of the Python being used, for example: +``venv/lib/python3.6/site-packages/`` To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash + ./configure --dev python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ diff --git a/docs/source/skeleton/index.rst b/docs/source/skeleton/index.rst deleted file mode 100644 index f99cdec..0000000 --- a/docs/source/skeleton/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -# Docs Structure Guide -# Rst docs - https://docutils.sourceforge.io/docs/ref/rst/restructuredtext.html -# -# 1. Place docs in folders under source for different sections -# 2. Link them by adding individual index files in each section -# to the main index, and then files for each section to their -# respective index files. -# 3. Use `.. include` statements to include other .rst files -# or part of them, or use hyperlinks to a section of the docs, -# to get rid of repetition. -# https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment -# -# Note: Replace these guide/placeholder docs - -.. include:: ../../../README.rst From 5431ee548c5bbfaf289f93281611d61c777aa575 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 28 Apr 2022 12:43:20 -0700 Subject: [PATCH 53/62] Properly check for existance of thirdparty dir Signed-off-by: Jono Yang --- configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure b/configure index 715c99f..d1b4fda 100755 --- a/configure +++ b/configure @@ -55,7 +55,7 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin ################################ # Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org -if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then +if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty" fi PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" From c44c4424d7fa82723c2aac7f2a79f380411e1949 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Apr 2022 15:41:07 +0200 Subject: [PATCH 54/62] Improve GH action documentation Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 188497e..b0a8d97 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -1,4 +1,4 @@ -name: Release library as a PyPI wheel and sdist on tag +name: Release library as a PyPI wheel and sdist on GH release creation on: release: From 99ba101572144cc5e5d42f2136985eb91163a46a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 29 Apr 2022 15:50:02 +0200 Subject: [PATCH 55/62] Use Python 3.9 as a base for actions Signed-off-by: Philippe Ombredanne --- .github/workflows/docs-ci.yml | 2 +- .github/workflows/pypi-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 656d624..18a44aa 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.7] + python-version: [3.9] steps: - name: Checkout code diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index b0a8d97..3a4fe27 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Install pypa/build run: python -m pip install build --user - name: Build a binary wheel and a source tarball From 00f4fe76dad5f0fa8efc6768af99079389c583ac Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 29 Apr 2022 14:31:34 -0700 Subject: [PATCH 56/62] Remove variable from string in fetch_thirdparty.py * The variable `environment` is not used when fetching sdists Signed-off-by: Jono Yang --- etc/scripts/fetch_thirdparty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 22147b2..042266c 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -269,11 +269,11 @@ def fetch_thirdparty( if TRACE: if not fetched: print( - f" ====> Sdist already available: {name}=={version} on: {environment}" + f" ====> Sdist already available: {name}=={version}" ) else: print( - f" ====> Sdist fetched: {fetched} for {name}=={version} on: {environment}" + f" ====> Sdist fetched: {fetched} for {name}=={version}" ) except utils_thirdparty.DistributionNotFound as e: From 5d48c1cbb7262455cc2c51958833ddb9ecb2bbce Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 2 May 2022 11:33:50 +0200 Subject: [PATCH 57/62] Improve thirdparty scripts Ensure that site-package dir exists. Other minor adjustments from a scancode-toolkit release Signed-off-by: Philippe Ombredanne --- etc/scripts/fetch_thirdparty.py | 7 ++-- etc/scripts/utils_requirements.py | 3 ++ etc/scripts/utils_thirdparty.py | 66 +++++++++++++++++++------------ 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 042266c..f31e81f 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -18,7 +18,8 @@ import utils_thirdparty import utils_requirements -TRACE = True +TRACE = False +TRACE_DEEP = False @click.command() @@ -204,7 +205,7 @@ def fetch_thirdparty( existing_wheels = None if existing_wheels: - if TRACE: + if TRACE_DEEP: print( f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" ) @@ -213,7 +214,7 @@ def fetch_thirdparty( else: continue - if TRACE: + if TRACE_DEEP: print(f"Fetching wheel for: {name}=={version} on: {environment}") try: diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index fbc456d..069b465 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,6 +8,7 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import os import re import subprocess @@ -110,6 +111,8 @@ def get_installed_reqs(site_packages_dir): Return the installed pip requirements as text found in `site_packages_dir` as a text. """ + if not os.path.exists(site_packages_dir): + raise Exception(f"site_packages directort: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 829cf8c..4c40969 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -8,8 +8,8 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -from collections import defaultdict import email +import functools import itertools import os import re @@ -18,6 +18,8 @@ import tempfile import time import urllib +from collections import defaultdict +from urllib.parse import quote_plus import attr import license_expression @@ -29,7 +31,6 @@ from commoncode.text import python_safe_name from packaging import tags as packaging_tags from packaging import version as packaging_version -from urllib.parse import quote_plus import utils_pip_compatibility_tags from utils_requirements import load_requirements @@ -111,7 +112,7 @@ """ -TRACE = True +TRACE = False TRACE_DEEP = False TRACE_ULTRA_DEEP = False @@ -233,7 +234,7 @@ def download_wheel( tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) """ if TRACE_DEEP: - print(f" download_wheel: {name}=={version}: {environment}") + print(f" download_wheel: {name}=={version}: {environment} and index_urls: {index_urls}") fetched_wheel_filenames = [] existing_wheel_filenames = [] @@ -311,6 +312,7 @@ def download_sdist( raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") +@functools.cache def get_package_versions( name, version=None, @@ -321,15 +323,28 @@ def get_package_versions( repository ``index_urls`` list of URLs. If ``version`` is not provided, return the latest available versions. """ + found = [] + not_found = [] for index_url in index_urls: try: repo = get_pypi_repo(index_url) package = repo.get_package(name, version) + if package: - yield package + found.append((package, index_url)) + else: + not_found.append((name, version, index_url)) except RemoteNotFetchedException as e: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + if TRACE_ULTRA_DEEP: + print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") + not_found.append((name, version, index_url)) + + if not found: + raise Exception(f"No PyPI package {name} @ {version} found!") + for package, index_url in found: + print(f"Fetched PyPI package {package.name} @ {package.version} info from {index_url}") + yield package ################################################################################ # @@ -546,14 +561,14 @@ def get_best_download_url( If none is found, return a synthetic remote URL. """ for index_url in index_urls: - pypi_package = get_pypi_package( + pypi_package = get_pypi_package_data( name=self.normalized_name, version=self.version, index_url=index_url, ) if pypi_package: if isinstance(pypi_package, tuple): - raise Exception("############", repr(pypi_package)) + raise Exception("############", repr(pypi_package), self.normalized_name, self.version, index_url) try: pypi_url = pypi_package.get_url_for_filename(self.filename) except Exception as e: @@ -1450,7 +1465,7 @@ def get_name_version(cls, name, version, packages): nvs = [p for p in cls.get_versions(name, packages) if p.version == version] if not nvs: - return name, version + return if len(nvs) == 1: return nvs[0] @@ -1494,8 +1509,8 @@ def dists_from_paths_or_urls(cls, paths_or_urls): >>> assert expected == result """ dists = [] - if TRACE_DEEP: - print(" ###paths_or_urls:", paths_or_urls) + if TRACE_ULTRA_DEEP: + print(" ###paths_or_urls:", paths_or_urls) installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: @@ -1618,7 +1633,6 @@ def tags(self): ) ) - ################################################################################ # # PyPI repo and link index for package wheels and sources @@ -1657,7 +1671,10 @@ def get_versions(self, name): The list may be empty. """ name = name and NameVer.normalize_name(name) - self._populate_links_and_packages(name) + try: + self._populate_links_and_packages(name) + except Exception as e: + print(f" ==> Cannot find versions of {name}: {e}") return self.packages_by_normalized_name.get(name, []) def get_latest_version(self, name): @@ -1703,7 +1720,7 @@ def _fetch_links(self, name, _LINKS={}): _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] links = _LINKS[index_url] - if TRACE_DEEP: + if TRACE_ULTRA_DEEP: print(f" Found links {links!r}") return links @@ -1803,7 +1820,6 @@ def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): _LINKS_REPO[url] = cls(url=url) return _LINKS_REPO[url] - ################################################################################ # Globals for remote repos to be lazily created and cached on first use for the # life of the session together with some convenience functions. @@ -1824,19 +1840,21 @@ def get_pypi_repo(index_url, _PYPI_REPO={}): return _PYPI_REPO[index_url] -def get_pypi_package(name, version, index_url, verbose=TRACE_DEEP): +@functools.cache +def get_pypi_package_data(name, version, index_url, verbose=TRACE_DEEP): """ Return a PypiPackage or None. """ try: + if verbose: + print(f" get_pypi_package_data: Fetching {name} @ {version} info from {index_url}") package = get_pypi_repo(index_url).get_package(name, version) if verbose: - print(f" get_pypi_package: {name} @ {version} info from {index_url}: {package}") + print(f" get_pypi_package_data: Fetched: {package}") return package except RemoteNotFetchedException as e: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - + print(f" get_pypi_package_data: Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") ################################################################################ # @@ -1998,7 +2016,6 @@ def fetch_and_save_path_or_url( fo.write(content) return content - ################################################################################ # Requirements processing ################################################################################ @@ -2036,7 +2053,6 @@ def get_required_packages( print(" get_required_packages: name:", name, "version:", version) yield repo.get_package(name, version) - ################################################################################ # Functions to update or fetch ABOUT and license files ################################################################################ @@ -2115,7 +2131,6 @@ def get_other_dists(_package, _dist): for p in packages_by_name[local_package.name] if p.version != local_package.version ] - other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: latest_local_dists = list(other_local_version.get_distributions()) @@ -2187,7 +2202,6 @@ def get_other_dists(_package, _dist): lic_errs = "\n".join(lic_errs) print(f"Failed to fetch some licenses:: {lic_errs}") - ################################################################################ # # Functions to build new Python wheels including native on multiple OSes @@ -2318,9 +2332,9 @@ def build_wheels_locally_if_pure_python( "--wheel-dir", wheel_dir, ] - + deps - + verbose - + [requirements_specifier] + +deps + +verbose + +[requirements_specifier] ) print(f"Building local wheels for: {requirements_specifier}") From 6a3c5b0b9e351b9c8730836c2db878a1540cbe2a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 7 May 2022 19:44:52 +0200 Subject: [PATCH 58/62] Update thirdparty fetching utilities These were buggy in some corner cases. They have been updated such that: * --latest-version works. * we can reliable fetch combinations of wheels and sdists for multiple OS combos at once * we now support macOS universal wheels (for ARM CPUs) Caching is now simpler: we have essentially a single file-based cache under .cache. PyPI indexes are fetched and not cached, unless the new --use-cached-index is used which can be useful when fetching many thirdparty in a short timeframe. The first PyPI repository in a list has precendence and we never fetch from other repositories if we find wheels and sdsists there. This avoid pounding too much on the self-hosted repo. Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 6 +- etc/scripts/fetch_thirdparty.py | 188 +++---- etc/scripts/gen_pypi_simple.py | 22 +- etc/scripts/requirements.txt | 5 +- etc/scripts/utils_requirements.py | 19 +- etc/scripts/utils_thirdparty.py | 899 +++++++++++++----------------- 6 files changed, 480 insertions(+), 659 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 0f04b34..b052f25 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,7 +16,7 @@ @click.command() @click.option( "-d", - "--dest_dir", + "--dest", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", @@ -35,7 +35,7 @@ ) @click.help_option("-h", "--help") def check_thirdparty_dir( - dest_dir, + dest, wheels, sdists, ): @@ -45,7 +45,7 @@ def check_thirdparty_dir( # check for problems print(f"==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( - dest_dir=dest_dir, + dest_dir=dest, report_missing_sources=sdists, report_missing_wheels=wheels, ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index f31e81f..26d520f 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -100,11 +100,17 @@ "index_urls", type=str, metavar="INDEX", - default=utils_thirdparty.PYPI_INDEXES, + default=utils_thirdparty.PYPI_INDEX_URLS, show_default=True, multiple=True, help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", ) +@click.option( + "--use-cached-index", + is_flag=True, + help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", +) + @click.help_option("-h", "--help") def fetch_thirdparty( requirements_files, @@ -116,9 +122,10 @@ def fetch_thirdparty( wheels, sdists, index_urls, + use_cached_index, ): """ - Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions, + Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, and their ABOUT metadata, license and notices files. Download the PyPI packages listed in the combination of: @@ -126,16 +133,23 @@ def fetch_thirdparty( - the pip name==version --specifier SPECIFIER(s) - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. - Download wheels with the --wheels option for the ``--python-version`` PYVER(s) - and ``--operating_system`` OS(s) combinations defaulting to all supported combinations. + Download wheels with the --wheels option for the ``--python-version`` + PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all + supported combinations. Download sdists tarballs with the --sdists option. - Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched. + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels + and sources fetched. - Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs. + Download from the provided PyPI simple --index-url INDEX(s) URLs. """ + if not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + existing_packages_by_nv = { (package.name, package.version): package for package in utils_thirdparty.get_local_packages(directory=dest_dir) @@ -151,134 +165,88 @@ def fetch_thirdparty( required_name_versions.update(nvs) for specifier in specifiers: - nv = utils_requirements.get_name_version( + nv = utils_requirements.get_required_name_version( requirement=specifier, with_unpinned=latest_version, ) required_name_versions.add(nv) + if latest_version: + names = set(name for name, _version in sorted(required_name_versions)) + required_name_versions = {(n, None) for n in names} + if not required_name_versions: print("Error: no requirements requested.") sys.exit(1) - if not os.listdir(dest_dir) and not (wheels or sdists): - print("Error: one or both of --wheels and --sdists is required.") - sys.exit(1) - - if latest_version: - latest_name_versions = set() - names = set(name for name, _version in sorted(required_name_versions)) - for name in sorted(names): - latests = utils_thirdparty.PypiPackage.sorted( - utils_thirdparty.get_package_versions( - name=name, version=None, index_urls=index_urls - ) - ) - if not latests: - print(f"No distribution found for: {name}") - continue - latest = latests[-1] - latest_name_versions.add((latest.name, latest.version)) - required_name_versions = latest_name_versions - - if TRACE: - print("required_name_versions:", required_name_versions) + if TRACE_DEEP: + print("required_name_versions:") + for n, v in required_name_versions: + print(f" {n} @ {v}") + # create the environments matrix we need for wheels + environments = None if wheels: - # create the environments matrix we need for wheels evts = itertools.product(python_versions, operating_systems) environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - wheels_not_found = {} - sdists_not_found = {} - # iterate over requirements, one at a time + # Collect PyPI repos + repos = [] + for index_url in index_urls: + index_url = index_url.strip("/") + existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) + if existing: + existing.use_cached_index = use_cached_index + repos.append(existing) + else: + repo = utils_thirdparty.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + ) + repos.append(repo) + + wheels_fetched = [] + wheels_not_found = [] + + sdists_fetched = [] + sdists_not_found = [] + for name, version in sorted(required_name_versions): nv = name, version - existing_package = existing_packages_by_nv.get(nv) + print(f"Processing: {name} @ {version}") if wheels: for environment in environments: - if existing_package: - existing_wheels = list( - existing_package.get_supported_wheels(environment=environment) - ) - else: - existing_wheels = None - - if existing_wheels: - if TRACE_DEEP: - print( - f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" - ) - if all(w.is_pure() for w in existing_wheels): - break - else: - continue - - if TRACE_DEEP: - print(f"Fetching wheel for: {name}=={version} on: {environment}") - - try: - ( - fetched_wheel_filenames, - existing_wheel_filenames, - ) = utils_thirdparty.download_wheel( - name=name, - version=version, - environment=environment, - dest_dir=dest_dir, - index_urls=index_urls, - ) - if TRACE: - if existing_wheel_filenames: - print( - f" ====> Wheels already available: {name}=={version} on: {environment}" - ) - for whl in existing_wheel_filenames: - print(f" {whl}") - if fetched_wheel_filenames: - print(f" ====> Wheels fetched: {name}=={version} on: {environment}") - for whl in fetched_wheel_filenames: - print(f" {whl}") - - fwfns = fetched_wheel_filenames + existing_wheel_filenames - - if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns): - break - - except utils_thirdparty.DistributionNotFound as e: - wheels_not_found[f"{name}=={version}"] = str(e) - - if sdists: - if existing_package and existing_package.sdist: if TRACE: - print( - f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}" - ) - continue - - if TRACE: - print(f" Fetching sdist for: {name}=={version}") - - try: - fetched = utils_thirdparty.download_sdist( + print(f" ==> Fetching wheel for envt: {environment}") + fwfns = utils_thirdparty.download_wheel( name=name, version=version, + environment=environment, dest_dir=dest_dir, - index_urls=index_urls, + repos=repos, ) + if fwfns: + wheels_fetched.extend(fwfns) + else: + wheels_not_found.append(f"{name}=={version} for: {environment}") + if TRACE: + print(f" NOT FOUND") + if sdists: + if TRACE: + print(f" ==> Fetching sdist: {name}=={version}") + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + repos=repos, + ) + if fetched: + sdists_fetched.append(fetched) + else: + sdists_not_found.append(f"{name}=={version}") if TRACE: - if not fetched: - print( - f" ====> Sdist already available: {name}=={version}" - ) - else: - print( - f" ====> Sdist fetched: {fetched} for {name}=={version}" - ) - - except utils_thirdparty.DistributionNotFound as e: - sdists_not_found[f"{name}=={version}"] = str(e) + print(f" NOT FOUND") if wheels and wheels_not_found: print(f"==> MISSING WHEELS") @@ -291,7 +259,7 @@ def fetch_thirdparty( print(f" {sd}") print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir) + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 8de2b96..03312ab 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -25,26 +25,26 @@ class InvalidDistributionFilename(Exception): def get_package_name_from_filename(filename): """ - Return the package name extracted from a package ``filename``. - Optionally ``normalize`` the name according to distribution name rules. + Return the normalized package name extracted from a package ``filename``. + Normalization is done according to distribution name rules. Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") 'foo' - >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl") + >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl") 'foo-bar' >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") 'cython' >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") 'python-ldap' - >>> get_package_name_from_filename("foo.whl") - Traceback (most recent call last): - ... - InvalidDistributionFilename: ... - >>> get_package_name_from_filename("foo.png") - Traceback (most recent call last): - ... - InvalidFilePackageName: ... + >>> try: + ... get_package_name_from_filename("foo.whl") + ... except InvalidDistributionFilename: + ... pass + >>> try: + ... get_package_name_from_filename("foo.png") + ... except InvalidDistributionFilename: + ... pass """ if not filename or not filename.endswith(dist_exts): raise InvalidDistributionFilename(filename) diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt index 6591e49..ebb404b 100644 --- a/etc/scripts/requirements.txt +++ b/etc/scripts/requirements.txt @@ -1,12 +1,11 @@ aboutcode_toolkit -github-release-retry2 attrs commoncode click requests saneyaml -romp pip setuptools twine -wheel \ No newline at end of file +wheel +build \ No newline at end of file diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 069b465..7c99a33 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -8,7 +8,6 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import os import re import subprocess @@ -42,23 +41,23 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): print(f"Requirement line is not supported: ignored: {req_line}") continue - yield get_name_version(requirement=req_line, with_unpinned=with_unpinned) + yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) -def get_name_version(requirement, with_unpinned=False): +def get_required_name_version(requirement, with_unpinned=False): """ Return a (name, version) tuple given a`requirement` specifier string. Requirement version must be pinned. If ``with_unpinned`` is True, unpinned requirements are accepted and only the name portion is returned. For example: - >>> assert get_name_version("foo==1.2.3") == ("foo", "1.2.3") - >>> assert get_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") - >>> assert get_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") - >>> assert get_name_version("foo", with_unpinned=True) == ("foo", "") - >>> assert get_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_name_version("foo>=1.2") + >>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") >>> try: - ... assert not get_name_version("foo", with_unpinned=False) + ... assert not get_required_name_version("foo", with_unpinned=False) ... except Exception as e: ... assert "Requirement version must be pinned" in str(e) """ @@ -112,7 +111,7 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception(f"site_packages directort: {site_packages_dir!r} does not exists") + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4c40969..9cbda37 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -9,7 +9,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import email -import functools import itertools import os import re @@ -33,7 +32,6 @@ from packaging import version as packaging_version import utils_pip_compatibility_tags -from utils_requirements import load_requirements """ Utilities to manage Python thirparty libraries source, binaries and metadata in @@ -169,6 +167,16 @@ def get_python_dot_version(version): "macosx_10_15_x86_64", "macosx_11_0_x86_64", "macosx_11_intel", + "macosx_11_0_x86_64", + "macosx_11_intel", + "macosx_10_9_universal2", + "macosx_10_10_universal2", + "macosx_10_11_universal2", + "macosx_10_12_universal2", + "macosx_10_13_universal2", + "macosx_10_14_universal2", + "macosx_10_15_universal2", + "macosx_11_0_universal2", # 'macosx_11_0_arm64', ], "windows": [ @@ -179,18 +187,19 @@ def get_python_dot_version(version): THIRDPARTY_DIR = "thirdparty" CACHE_THIRDPARTY_DIR = ".cache/thirdparty" -ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" +################################################################################ +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" - PYPI_SIMPLE_URL = "https://pypi.org/simple" -PYPI_INDEXES = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) +PYPI_INDEX_URLS = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) + +################################################################################ EXTENSIONS_APP = (".pyz",) EXTENSIONS_SDIST = ( ".tar.gz", - ".tar.bz2", ".zip", ".tar.xz", ) @@ -217,134 +226,90 @@ class DistributionNotFound(Exception): pass -def download_wheel( - name, - version, - environment, - dest_dir=THIRDPARTY_DIR, - index_urls=PYPI_INDEXES, -): +def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tuple()): """ Download the wheels binary distribution(s) of package ``name`` and - ``version`` matching the ``environment`` Environment constraints from the - PyPI simple repository ``index_urls`` list of URLs into the ``dest_dir`` - directory. + ``version`` matching the ``environment`` Environment constraints into the + ``dest_dir`` directory. Return a list of fetched_wheel_filenames, possibly + empty. - Raise a DistributionNotFound if no wheel is not found. Otherwise, return a - tuple of lists of (fetched_wheel_filenames, existing_wheel_filenames) + Use the first PyPI simple repository from a list of ``repos`` that contains this wheel. """ if TRACE_DEEP: - print(f" download_wheel: {name}=={version}: {environment} and index_urls: {index_urls}") + print(f" download_wheel: {name}=={version} for envt: {environment}") - fetched_wheel_filenames = [] - existing_wheel_filenames = [] - try: - for pypi_package in get_package_versions( - name=name, - version=version, - index_urls=index_urls, - ): - if not pypi_package.wheels: - continue - - supported_wheels = list(pypi_package.get_supported_wheels(environment=environment)) - if not supported_wheels: - continue + if not repos: + repos = DEFAULT_PYPI_REPOS - for wheel in supported_wheels: - if os.path.exists(os.path.join(dest_dir, wheel.filename)): - # do not refetch - existing_wheel_filenames.append(wheel.filename) - continue + fetched_wheel_filenames = [] - if TRACE: - print(f" Fetching wheel from index: {wheel.download_url}") - fetched_wheel_filename = wheel.download(dest_dir=dest_dir) - fetched_wheel_filenames.add(fetched_wheel_filename) + for repo in repos: + package = repo.get_package_version(name=name, version=version) + if not package: + if TRACE_DEEP: + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") + continue + supported_wheels = list(package.get_supported_wheels(environment=environment)) + if not supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: No supported wheel for {name}=={version}: {environment} " + ) + continue - except Exception as e: - raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: {e}") from e + for wheel in supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" + ) + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.append(fetched_wheel_filename) - if not fetched_wheel_filenames and not existing_wheel_filenames: - raise DistributionNotFound(f"Failed to fetch wheel: {name}=={version}: No wheel found") + if fetched_wheel_filenames: + # do not futher fetch from other repos if we find in first, typically PyPI + break - return fetched_wheel_filenames, existing_wheel_filenames + return fetched_wheel_filenames -def download_sdist( - name, - version, - dest_dir=THIRDPARTY_DIR, - index_urls=PYPI_INDEXES, -): +def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): """ Download the sdist source distribution of package ``name`` and ``version`` - from the PyPI simple repository ``index_urls`` list of URLs into the - ``dest_dir`` directory. + into the ``dest_dir`` directory. Return a fetched filename or None. - Raise a DistributionNotFound if this was not found. Return the filename if - downloaded and False if not downloaded because it already exists. + Use the first PyPI simple repository from a list of ``repos`` that contains + this sdist. """ - if TRACE_DEEP: - print(f"download_sdist: {name}=={version}: ") - - try: - for pypi_package in get_package_versions( - name=name, - version=version, - index_urls=index_urls, - ): - if not pypi_package.sdist: - continue - - if os.path.exists(os.path.join(dest_dir, pypi_package.sdist.filename)): - # do not refetch - return False - if TRACE: - print(f" Fetching sources from index: {pypi_package.sdist.download_url}") - fetched = pypi_package.sdist.download(dest_dir=dest_dir) - if fetched: - return pypi_package.sdist.filename + if TRACE: + print(f" download_sdist: {name}=={version}") - except Exception as e: - raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: {e}") from e + if not repos: + repos = DEFAULT_PYPI_REPOS - raise DistributionNotFound(f"Failed to fetch sdist: {name}=={version}: No sources found") + fetched_sdist_filename = None + for repo in repos: + package = repo.get_package_version(name=name, version=version) -@functools.cache -def get_package_versions( - name, - version=None, - index_urls=PYPI_INDEXES, -): - """ - Yield PypiPackages with ``name`` and ``version`` from the PyPI simple - repository ``index_urls`` list of URLs. - If ``version`` is not provided, return the latest available versions. - """ - found = [] - not_found = [] - for index_url in index_urls: - try: - repo = get_pypi_repo(index_url) - package = repo.get_package(name, version) + if not package: + if TRACE_DEEP: + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") + continue + sdist = package.sdist + if not sdist: + if TRACE_DEEP: + print(f" download_sdist: No sdist for {name}=={version}") + continue - if package: - found.append((package, index_url)) - else: - not_found.append((name, version, index_url)) - except RemoteNotFetchedException as e: - if TRACE_ULTRA_DEEP: - print(f"Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - not_found.append((name, version, index_url)) + if TRACE_DEEP: + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) - if not found: - raise Exception(f"No PyPI package {name} @ {version} found!") + if fetched_sdist_filename: + # do not futher fetch from other repos if we find in first, typically PyPI + break - for package, index_url in found: - print(f"Fetched PyPI package {package.name} @ {package.version} info from {index_url}") - yield package + return fetched_sdist_filename ################################################################################ # @@ -377,17 +342,6 @@ def normalize_name(name): """ return name and re.sub(r"[-_.]+", "-", name).lower() or name - @staticmethod - def standardize_name(name): - """ - Return a standardized package name, e.g. lowercased and using - not _ - """ - return name and re.sub(r"[-_]+", "-", name).lower() or name - - @property - def name_ver(self): - return f"{self.name}-{self.version}" - def sortable_name_version(self): """ Return a tuple of values to sort by name, then version. @@ -403,7 +357,7 @@ def sorted(cls, namevers): @attr.attributes class Distribution(NameVer): - # field names that can be updated from another dist of mapping + # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", "copyright", @@ -421,6 +375,13 @@ class Distribution(NameVer): metadata=dict(help="File name."), ) + path_or_url = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Path or URL"), + ) + sha256 = attr.ib( repr=False, type=str, @@ -545,36 +506,50 @@ def package_url(self): """ Return a Package URL string of self. """ - return str(packageurl.PackageURL(**self.purl_identifiers())) + return str( + packageurl.PackageURL( + type=self.type, + namespace=self.namespace, + name=self.name, + version=self.version, + subpath=self.subpath, + qualifiers=self.qualifiers, + ) + ) @property def download_url(self): return self.get_best_download_url() - def get_best_download_url( - self, - index_urls=tuple([PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL]), - ): + def get_best_download_url(self, repos=tuple()): """ - Return the best download URL for this distribution where best means that - PyPI is better and our selfhosted repo URLs are second. - If none is found, return a synthetic remote URL. + Return the best download URL for this distribution where best means this + is the first URL found for this distribution found in the list of + ``repos``. + + If none is found, return a synthetic PyPI remote URL. """ - for index_url in index_urls: - pypi_package = get_pypi_package_data( - name=self.normalized_name, - version=self.version, - index_url=index_url, - ) - if pypi_package: - if isinstance(pypi_package, tuple): - raise Exception("############", repr(pypi_package), self.normalized_name, self.version, index_url) - try: - pypi_url = pypi_package.get_url_for_filename(self.filename) - except Exception as e: - raise Exception(repr(pypi_package)) from e - if pypi_url: - return pypi_url + + if not repos: + repos = DEFAULT_PYPI_REPOS + + for repo in repos: + package = repo.get_package_version(name=self.name, version=self.version) + if not package: + if TRACE: + print( + f" get_best_download_url: {self.name}=={self.version} " + f"not found in {repo.index_url}" + ) + continue + pypi_url = package.get_url_for_filename(self.filename) + if pypi_url: + return pypi_url + else: + if TRACE: + print( + f" get_best_download_url: {self.filename} not found in {repo.index_url}" + ) def download(self, dest_dir=THIRDPARTY_DIR): """ @@ -582,16 +557,17 @@ def download(self, dest_dir=THIRDPARTY_DIR): Return the fetched filename. """ assert self.filename - if TRACE: + if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", self.filename, ) - fetch_and_save_path_or_url( - filename=self.filename, - dest_dir=dest_dir, + # FIXME: + fetch_and_save( path_or_url=self.path_or_url, + dest_dir=dest_dir, + filename=self.filename, as_text=False, ) return self.filename @@ -616,7 +592,7 @@ def notice_download_url(self): def from_path_or_url(cls, path_or_url): """ Return a distribution built from the data found in the filename of a - `path_or_url` string. Raise an exception if this is not a valid + ``path_or_url`` string. Raise an exception if this is not a valid filename. """ filename = os.path.basename(path_or_url.strip("/")) @@ -647,47 +623,6 @@ def from_filename(cls, filename): clazz = cls.get_dist_class(filename) return clazz.from_filename(filename) - def purl_identifiers(self, skinny=False): - """ - Return a mapping of non-empty identifier name/values for the purl - fields. If skinny is True, only inlucde type, namespace and name. - """ - identifiers = dict( - type=self.type, - namespace=self.namespace, - name=self.name, - ) - - if not skinny: - identifiers.update( - version=self.version, - subpath=self.subpath, - qualifiers=self.qualifiers, - ) - - return {k: v for k, v in sorted(identifiers.items()) if v} - - def identifiers(self, purl_as_fields=True): - """ - Return a mapping of non-empty identifier name/values. - Return each purl fields separately if purl_as_fields is True. - Otherwise return a package_url string for the purl. - """ - if purl_as_fields: - identifiers = self.purl_identifiers() - else: - identifiers = dict(package_url=self.package_url) - - identifiers.update( - download_url=self.download_url, - filename=self.filename, - md5=self.md5, - sha1=self.sha1, - package_url=self.package_url, - ) - - return {k: v for k, v in sorted(identifiers.items()) if v} - def has_key_metadata(self): """ Return True if this distribution has key metadata required for basic attribution. @@ -817,7 +752,7 @@ def load_remote_about_data(self): NOTICE file if any. Return True if the data was updated. """ try: - about_text = fetch_content_from_path_or_url_through_cache( + about_text = CACHE.get( path_or_url=self.about_download_url, as_text=True, ) @@ -831,7 +766,7 @@ def load_remote_about_data(self): notice_file = about_data.pop("notice_file", None) if notice_file: try: - notice_text = fetch_content_from_path_or_url_through_cache( + notice_text = CACHE.get( path_or_url=self.notice_download_url, as_text=True, ) @@ -882,12 +817,12 @@ def get_license_keys(self): return ["unknown"] return keys - def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): + def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url().links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -902,10 +837,10 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): # try remotely first lic_url = get_license_link_for_filename(filename=filename, urls=urls) - fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + fetch_and_save( path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, as_text=True, ) if TRACE: @@ -915,10 +850,10 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR): try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" - fetch_and_save_path_or_url( - filename=filename, - dest_dir=dest_dir, + fetch_and_save( path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, as_text=True, ) if TRACE: @@ -1077,6 +1012,84 @@ class InvalidDistributionFilename(Exception): pass +def get_sdist_name_ver_ext(filename): + """ + Return a (name, version, extension) if filename is a valid sdist name. Some legacy + binary builds have weird names. Return False otherwise. + + In particular they do not use PEP440 compliant versions and/or mix tags, os + and arch names in tarball names and versions: + + >>> assert get_sdist_name_ver_ext("intbitset-1.3.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-1.3.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.zip") + >>> assert not get_sdist_name_ver_ext("intbitset-2.0.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-2.0.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.src.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.x86_64.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1.linux-x86_64.tar.gz") + >>> assert not get_sdist_name_ver_ext("cffi-1.2.0-1.tar.gz") + >>> assert not get_sdist_name_ver_ext("html5lib-1.0-reupload.tar.gz") + >>> assert not get_sdist_name_ver_ext("selenium-2.0-dev-9429.tar.gz") + >>> assert not get_sdist_name_ver_ext("testfixtures-1.8.0dev-r4464.tar.gz") + """ + name_ver = None + extension = None + + for ext in EXTENSIONS_SDIST: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + return False + + name, _, version = name_ver.rpartition("-") + + if not name or not version: + return False + + # weird version + if any( + w in version + for w in ( + "x86_64", + "i386", + ) + ): + return False + + # all char versions + if version.isalpha(): + return False + + # non-pep 440 version + if "-" in version: + return False + + # single version + if version.isdigit() and len(version) == 1: + return False + + # r1 version + if len(version) == 2 and version[0]=="r" and version[1].isdigit(): + return False + + # dotless version (but calver is OK) + if "." not in version and len(version) < 3: + return False + + # version with dashes selenium-2.0-dev-9429.tar.gz + if name.endswith(("dev",)) and "." not in version: + return False + # version pre or post, old legacy + if version.startswith(("beta", "rc", "pre", "post", "final")): + return False + + return name, version, extension + + @attr.attributes class Sdist(Distribution): @@ -1093,21 +1106,11 @@ def from_filename(cls, filename): Return a Sdist object built from a filename. Raise an exception if this is not a valid sdist filename """ - name_ver = None - extension = None - - for ext in EXTENSIONS_SDIST: - if filename.endswith(ext): - name_ver, extension, _ = filename.rpartition(ext) - break - - if not extension or not name_ver: + name_ver_ext = get_sdist_name_ver_ext(filename) + if not name_ver_ext: raise InvalidDistributionFilename(filename) - name, _, version = name_ver.rpartition("-") - - if not name or not version: - raise InvalidDistributionFilename(filename) + name, version, extension = name_ver_ext return cls( type="pypi", @@ -1295,8 +1298,8 @@ def is_pure_wheel(filename): @attr.attributes class PypiPackage(NameVer): """ - A Python package with its "distributions", e.g. wheels and source - distribution , ABOUT files and licenses or notices. + A Python package contains one or more wheels and one source distribution + from a repository. """ sdist = attr.ib( @@ -1313,16 +1316,6 @@ class PypiPackage(NameVer): metadata=dict(help="List of Wheel for this package"), ) - @property - def specifier(self): - """ - A requirement specifier for this package - """ - if self.version: - return f"{self.name}=={self.version}" - else: - return self.name - def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): """ Yield all the Wheel of this package supported and compatible with the @@ -1404,17 +1397,20 @@ def packages_from_dir(cls, directory): Yield PypiPackages built from files found in at directory path. """ base = os.path.abspath(directory) + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) - return cls.packages_from_many_paths_or_urls(paths) + return PypiPackage.packages_from_many_paths_or_urls(paths) @classmethod def packages_from_many_paths_or_urls(cls, paths_or_urls): """ Yield PypiPackages built from a list of paths or URLs. + These are sorted by name and then by version from oldest to newest. """ - dists = cls.dists_from_paths_or_urls(paths_or_urls) + dists = PypiPackage.dists_from_paths_or_urls(paths_or_urls) if TRACE_ULTRA_DEEP: print("packages_from_many_paths_or_urls: dists:", dists) @@ -1429,54 +1425,11 @@ def packages_from_many_paths_or_urls(cls, paths_or_urls): print("packages_from_many_paths_or_urls", package) yield package - @classmethod - def get_versions(cls, name, packages): - """ - Return a subset list of package versions from a list of `packages` that - match PypiPackage `name`. - The list is sorted by version from oldest to most recent. - """ - norm_name = NameVer.normalize_name(name) - versions = [p for p in packages if p.normalized_name == norm_name] - return cls.sorted(versions) - - @classmethod - def get_latest_version(cls, name, packages): - """ - Return the latest version of PypiPackage `name` from a list of `packages`. - """ - versions = cls.get_versions(name, packages) - if not versions: - return - return versions[-1] - - @classmethod - def get_name_version(cls, name, version, packages): - """ - Return the PypiPackage with `name` and `version` from a list of `packages` - or None if it is not found. - If `version` is None, return the latest version found. - """ - if TRACE_ULTRA_DEEP: - print("get_name_version:", name, version, packages) - if not version: - return cls.get_latest_version(name, packages) - - nvs = [p for p in cls.get_versions(name, packages) if p.version == version] - - if not nvs: - return - - if len(nvs) == 1: - return nvs[0] - - raise Exception(f"More than one PypiPackage with {name}=={version}") - @classmethod def dists_from_paths_or_urls(cls, paths_or_urls): """ Return a list of Distribution given a list of - `paths_or_urls` to wheels or source distributions. + ``paths_or_urls`` to wheels or source distributions. Each Distribution receives two extra attributes: - the path_or_url it was created from @@ -1488,25 +1441,20 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl ... https://example.com/bar/bitarray-0.8.1.tar.gz - ... bitarray-0.8.1.tar.gz.ABOUT bit.LICENSE'''.split() - >>> result = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) + ... bitarray-0.8.1.tar.gz.ABOUT + ... bit.LICENSE'''.split() + >>> results = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) >>> for r in results: - ... r.filename = '' - ... r.path_or_url = '' - >>> expected = [ - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['linux_x86_64']), - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']), - ... Wheel(name='bitarray', version='0.8.1', build='', - ... python_versions=['cp36'], abis=['cp36m'], - ... platforms=['win_amd64']), - ... Sdist(name='bitarray', version='0.8.1'), - ... Sdist(name='bitarray', version='0.8.1') - ... ] - >>> assert expected == result + ... print(r.__class__.__name__, r.name, r.version) + ... if isinstance(r, Wheel): + ... print(" ", ", ".join(r.python_versions), ", ".join(r.platforms)) + Wheel bitarray 0.8.1 + cp36 linux_x86_64 + Wheel bitarray 0.8.1 + cp36 macosx_10_9_x86_64, macosx_10_10_x86_64 + Wheel bitarray 0.8.1 + cp36 win_amd64 + Sdist bitarray 0.8.1 """ dists = [] if TRACE_ULTRA_DEEP: @@ -1518,7 +1466,14 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists.append(dist) if TRACE_DEEP: print( - " ===> dists_from_paths_or_urls:", dist, "with URL:", dist.download_url + " ===> dists_from_paths_or_urls:", + dist, + "\n ", + "with URL:", + dist.download_url, + "\n ", + "from URL:", + path_or_url, ) except InvalidDistributionFilename: if TRACE_DEEP: @@ -1653,101 +1608,105 @@ class PypiSimpleRepository: metadata=dict(help="Base PyPI simple URL for this index."), ) - packages_by_normalized_name = attr.ib( + # we keep a nested mapping of PypiPackage that has this shape: + # {name: {version: PypiPackage, version: PypiPackage, etc} + # the inner versions mapping is sorted by version from oldest to newest + + packages = attr.ib( type=dict, - default=attr.Factory(lambda: defaultdict(list)), - metadata=dict(help="Mapping of {package name: [package objects]} available in this repo"), + default=attr.Factory(lambda: defaultdict(dict)), + metadata=dict( + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + ), ) - packages_by_normalized_name_version = attr.ib( - type=dict, - default=attr.Factory(dict), - metadata=dict(help="Mapping of {(name, version): package object} available in this repo"), + fetched_package_normalized_names = attr.ib( + type=set, + default=attr.Factory(set), + metadata=dict(help="A set of already fetched package normalized names."), ) - def get_versions(self, name): + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict(help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."), + ) + + def _get_package_versions_map(self, name): """ - Return a list of all available PypiPackage version for this package name. - The list may be empty. + Return a mapping of all available PypiPackage version for this package name. + The mapping may be empty. It is ordered by version from oldest to newest """ - name = name and NameVer.normalize_name(name) - try: - self._populate_links_and_packages(name) - except Exception as e: - print(f" ==> Cannot find versions of {name}: {e}") - return self.packages_by_normalized_name.get(name, []) + assert name + normalized_name = NameVer.normalize_name(name) + versions = self.packages[normalized_name] + if not versions and normalized_name not in self.fetched_package_normalized_names: + self.fetched_package_normalized_names.add(normalized_name) + try: + links = self.fetch_links(normalized_name=normalized_name) + # note that thsi is sorted so the mapping is also sorted + versions = { + package.version: package + for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) + } + self.packages[normalized_name] = versions + except RemoteNotFetchedException as e: + if TRACE: + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + + if not versions and TRACE: + print(f"WARNING: package {name} not found in repo: {self.index_url}") + + return versions - def get_latest_version(self, name): + def get_package_versions(self, name): """ - Return the latest PypiPackage version for this package name or None. + Return a mapping of all available PypiPackage version as{version: + package} for this package name. The mapping may be empty but not None. + It is sorted by version from oldest to newest. """ - versions = self.get_versions(name) - return PypiPackage.get_latest_version(name, versions) + return dict(self._get_package_versions_map(name)) - def get_package(self, name, version): + def get_package_version(self, name, version=None): """ Return the PypiPackage with name and version or None. + Return the latest PypiPackage version if version is None. """ - versions = self.get_versions(name) - if TRACE_DEEP: - print("PypiPackage.get_package:versions:", versions) - return PypiPackage.get_name_version(name, version, versions) + if not version: + versions = list(self._get_package_versions_map(name).values()) + return versions and versions[-1] + else: + return self._get_package_versions_map(name).get(version) - def _fetch_links(self, name, _LINKS={}): + def fetch_links(self, normalized_name): """ Return a list of download link URLs found in a PyPI simple index for package name using the `index_url` of this repository. """ - name = name and NameVer.normalize_name(name) - index_url = self.index_url - - name = name and NameVer.normalize_name(name) - index_url = index_url.strip("/") - index_url = f"{index_url}/{name}" - - if TRACE_DEEP: - print( - f" Finding links for {name!r} from PyPI index: {index_url} : cached?:", - index_url in _LINKS, - ) - - if index_url not in _LINKS: - text = fetch_content_from_path_or_url_through_cache(path_or_url=index_url, as_text=True) - links = collect_urls(text) - # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] - links = [url for url, _, _sha256 in links] - _LINKS[index_url] = [l for l in links if l.endswith(EXTENSIONS)] - - links = _LINKS[index_url] - if TRACE_ULTRA_DEEP: - print(f" Found links {links!r}") + package_url = f"{self.index_url}/{normalized_name}" + text = CACHE.get( + path_or_url=package_url, + as_text=True, + force=not self.use_cached_index, + ) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] return links - def _populate_links_and_packages(self, name): - name = name and NameVer.normalize_name(name) - - if TRACE_DEEP: - print("PypiPackage._populate_links_and_packages:name:", name) - - links = self._fetch_links(name) - packages = list(PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links)) - - if TRACE_DEEP: - print("PypiPackage._populate_links_and_packages:packages:", packages) - self.packages_by_normalized_name[name] = packages - - for p in packages: - name = name and NameVer.normalize_name(p.name) - self.packages_by_normalized_name_version[(name, p.version)] = p +PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL) +PYPI_SELFHOSTED_REPO = PypiSimpleRepository(index_url=ABOUT_PYPI_SIMPLE_URL) +DEFAULT_PYPI_REPOS = PYPI_PUBLIC_REPO, PYPI_SELFHOSTED_REPO +DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} @attr.attributes class LinksRepository: """ - Represents a simple links repository such an HTTP directory listing or a - page with links. + Represents a simple links repository such an HTTP directory listing or an + HTML page with links. """ url = attr.ib( @@ -1762,14 +1721,23 @@ class LinksRepository: metadata=dict(help="List of links available in this repo"), ) + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict(help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."), + ) + def __attrs_post_init__(self): if not self.links: self.links = self.find_links() - def find_links(self): + def find_links(self, _CACHE=[]): """ Return a list of link URLs found in the HTML page at `self.url` """ + if _CACHE: + return _CACHE + links_url = self.url if TRACE_DEEP: print(f"Finding links from: {links_url}") @@ -1781,9 +1749,10 @@ def find_links(self): if TRACE_DEEP: print(f"Base URL {base_url}") - text = fetch_content_from_path_or_url_through_cache( + text = CACHE.get( path_or_url=links_url, as_text=True, + force=not self.use_cached_index, ) links = [] @@ -1812,12 +1781,13 @@ def find_links(self): if TRACE: print(f"Found {len(links)} links at {links_url}") + _CACHE.extend(links) return links @classmethod - def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}): + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): if url not in _LINKS_REPO: - _LINKS_REPO[url] = cls(url=url) + _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) return _LINKS_REPO[url] ################################################################################ @@ -1833,29 +1803,6 @@ def get_local_packages(directory=THIRDPARTY_DIR): """ return list(PypiPackage.packages_from_dir(directory=directory)) - -def get_pypi_repo(index_url, _PYPI_REPO={}): - if index_url not in _PYPI_REPO: - _PYPI_REPO[index_url] = PypiSimpleRepository(index_url=index_url) - return _PYPI_REPO[index_url] - - -@functools.cache -def get_pypi_package_data(name, version, index_url, verbose=TRACE_DEEP): - """ - Return a PypiPackage or None. - """ - try: - if verbose: - print(f" get_pypi_package_data: Fetching {name} @ {version} info from {index_url}") - package = get_pypi_repo(index_url).get_package(name, version) - if verbose: - print(f" get_pypi_package_data: Fetched: {package}") - return package - - except RemoteNotFetchedException as e: - print(f" get_pypi_package_data: Failed to fetch PyPI package {name} @ {version} info from {index_url}: {e}") - ################################################################################ # # Basic file and URL-based operations using a persistent file-based Cache @@ -1875,34 +1822,40 @@ class Cache: def __attrs_post_init__(self): os.makedirs(self.directory, exist_ok=True) - def clear(self): - shutil.rmtree(self.directory) - - def get(self, path_or_url, as_text=True): + def get(self, path_or_url, as_text=True, force=False): """ - Get a file from a `path_or_url` through the cache. - `path_or_url` can be a path or a URL to a file. + Return the content fetched from a ``path_or_url`` through the cache. + Raise an Exception on errors. Treats the content as text if as_text is + True otherwise as treat as binary. `path_or_url` can be a path or a URL + to a file. """ cache_key = quote_plus(path_or_url.strip("/")) cached = os.path.join(self.directory, cache_key) - if not os.path.exists(cached): + if force or not os.path.exists(cached): + if TRACE_DEEP: + print(f" FILE CACHE MISS: {path_or_url}") content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) return content else: + if TRACE_DEEP: + print(f" FILE CACHE HIT: {path_or_url}") return get_local_file_content(path=cached, as_text=as_text) +CACHE = Cache() + + def get_file_content(path_or_url, as_text=True): """ Fetch and return the content at `path_or_url` from either a local path or a remote URL. Return the content as bytes is `as_text` is False. """ if path_or_url.startswith("https://"): - if TRACE: + if TRACE_DEEP: print(f"Fetching: {path_or_url}") _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content @@ -1954,7 +1907,7 @@ def get_remote_file_content( # using a GET with stream=True ensure we get the the final header from # several redirects and that we can ignore content there. A HEAD request may # not get us this last header - print(f" DOWNLOADING {url}") + print(f" DOWNLOADING: {url}") with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA @@ -1978,35 +1931,19 @@ def get_remote_file_content( return response.headers, response.text if as_text else response.content -def fetch_content_from_path_or_url_through_cache( +def fetch_and_save( path_or_url, - as_text=True, - cache=Cache(), -): - """ - Return the content from fetching at path or URL. Raise an Exception on - errors. Treats the content as text if as_text is True otherwise as treat as - binary. Use the provided file cache. This is the main entry for using the - cache. - - Note: the `cache` argument is a global, though it does not really matter - since it does not hold any state which is only kept on disk. - """ - return cache.get(path_or_url=path_or_url, as_text=as_text) - - -def fetch_and_save_path_or_url( - filename, dest_dir, - path_or_url, + filename, as_text=True, ): """ - Return the content from fetching the `filename` file name at URL or path - and save to `dest_dir`. Raise an Exception on errors. Treats the content as - text if as_text is True otherwise as treat as binary. + Fetch content at ``path_or_url`` URL or path and save this to + ``dest_dir/filername``. Return the fetched content. Raise an Exception on + errors. Treats the content as text if as_text is True otherwise as treat as + binary. """ - content = fetch_content_from_path_or_url_through_cache( + content = CACHE.get( path_or_url=path_or_url, as_text=as_text, ) @@ -2017,44 +1954,9 @@ def fetch_and_save_path_or_url( return content ################################################################################ -# Requirements processing -################################################################################ - - -def get_required_remote_packages( - requirements_file="requirements.txt", - index_url=PYPI_SIMPLE_URL, -): - """ - Yield tuple of (name, version, PypiPackage) for packages listed in the - `requirements_file` requirements file and found in the PyPI index - ``index_url`` URL. - """ - required_name_versions = load_requirements(requirements_file=requirements_file) - return get_required_packages(required_name_versions=required_name_versions, index_url=index_url) - - -def get_required_packages( - required_name_versions, - index_url=PYPI_SIMPLE_URL, -): - """ - Yield tuple of (name, version) or a PypiPackage for package name/version - listed in the ``required_name_versions`` list and found in the PyPI index - ``index_url`` URL. - """ - if TRACE: - print("get_required_packages", index_url) - - repo = get_pypi_repo(index_url=index_url) - - for name, version in required_name_versions: - if TRACE: - print(" get_required_packages: name:", name, "version:", version) - yield repo.get_package(name, version) - -################################################################################ +# # Functions to update or fetch ABOUT and license files +# ################################################################################ @@ -2075,7 +1977,7 @@ def clean_about_files( local_dist.save_about_and_notice_files(dest_dir) -def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using best efforts: @@ -2085,6 +1987,8 @@ def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR): - derive from existing distribution with same name and latest version that would have such ABOUT file - extract ABOUT file data from distributions PKGINFO or METADATA files + + Use available existing on-disk cached index if use_cached_index is True. """ def get_other_dists(_package, _dist): @@ -2094,7 +1998,6 @@ def get_other_dists(_package, _dist): """ return [d for d in _package.get_distributions() if d != _dist] - selfhosted_repo = get_pypi_repo(index_url=ABOUT_PYPI_SIMPLE_URL) local_packages = get_local_packages(directory=dest_dir) packages_by_name = defaultdict(list) for local_package in local_packages: @@ -2110,7 +2013,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2122,7 +2025,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2148,7 +2051,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to fetch remotely @@ -2157,14 +2060,16 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version + # and that is in our self hosted repo + lpv = local_package.version + lpn = local_package.name + other_remote_packages = [ - p - for p in selfhosted_repo.get_versions(local_package.name) - if p.version != local_package.version + p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv ] latest_version = other_remote_packages and other_remote_packages[-1] @@ -2184,7 +2089,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get data from pkginfo (no license though) @@ -2194,7 +2099,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files(dest_dir) + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2305,66 +2210,16 @@ def download_wheels_with_pip( downloaded = existing ^ set(os.listdir(dest_dir)) return sorted(downloaded), error - -def build_wheels_locally_if_pure_python( - requirements_specifier, - with_deps=False, - verbose=False, - dest_dir=THIRDPARTY_DIR, -): - """ - Given pip `requirements_specifier` string (such as package names or as - name==version), build the corresponding binary wheel(s) locally. - - If all these are "pure" Python wheels that run on all Python 3 versions and - operating systems, copy them back in `dest_dir` if they do not exists there - - Return a tuple of (True if all wheels are "pure", list of built wheel file names) - """ - deps = [] if with_deps else ["--no-deps"] - verbose = ["--verbose"] if verbose else [] - - wheel_dir = tempfile.mkdtemp(prefix="scancode-release-wheels-local-") - cli_args = ( - [ - "pip", - "wheel", - "--wheel-dir", - wheel_dir, - ] - +deps - +verbose - +[requirements_specifier] - ) - - print(f"Building local wheels for: {requirements_specifier}") - print(f"Using command:", " ".join(cli_args)) - call(cli_args) - - built = os.listdir(wheel_dir) - if not built: - return [] - - all_pure = all(is_pure_wheel(bwfn) for bwfn in built) - - if not all_pure: - print(f" Some wheels are not pure") - - print(f" Copying local wheels") - pure_built = [] - for bwfn in built: - owfn = os.path.join(dest_dir, bwfn) - if not os.path.exists(owfn): - nwfn = os.path.join(wheel_dir, bwfn) - fileutils.copyfile(nwfn, owfn) - pure_built.append(bwfn) - print(f" Built local wheel: {bwfn}") - return all_pure, pure_built +################################################################################ +# +# Functions to check for problems +# +################################################################################ def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") From 0cdf006d099587092517228be0e3182651dd0a1c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 9 May 2022 12:40:59 +0200 Subject: [PATCH 59/62] Do not install patch.py by default It has deprecated regexes and needs some love to maintain Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 8 +++++++- configure | 2 +- configure.bat | 2 +- setup.cfg | 5 +++-- src/extractcode/archive.py | 8 +++++++- src/extractcode/patch.py | 3 +-- tests/test_patch.py | 6 ++++++ 7 files changed, 26 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e441787..106598c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,13 @@ Changelog ========= -v30.0.0 (next) +v31.0.0 +-------- + +- Do not install patch.py by default. Instead this is now an extra + + +v30.0.0 -------- - Update to the latest skeleton. The virtualenv is now created under the venv diff --git a/configure b/configure index e60b77a..319dffe 100755 --- a/configure +++ b/configure @@ -29,7 +29,7 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable .[full] --constraint requirements.txt" -DEV_REQUIREMENTS="--editable .[full,testing] --constraint requirements.txt --constraint requirements-dev.txt" +DEV_REQUIREMENTS="--editable .[full,testing,patch] --constraint requirements.txt --constraint requirements-dev.txt" DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv diff --git a/configure.bat b/configure.bat index 50bb0eb..4174d09 100644 --- a/configure.bat +++ b/configure.bat @@ -27,7 +27,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable .[full] --constraint requirements.txt" -set "DEV_REQUIREMENTS=--editable .[full,testing] --constraint requirements.txt --constraint requirements-dev.txt" +set "DEV_REQUIREMENTS=--editable .[full,testing,patch] --constraint requirements.txt --constraint requirements-dev.txt" set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv diff --git a/setup.cfg b/setup.cfg index 26655e6..1515c28 100644 --- a/setup.cfg +++ b/setup.cfg @@ -70,8 +70,6 @@ install_requires = commoncode >= 21.5.25 plugincode >= 21.1.21 typecode >= 21.6.1 - patch >= 1.16 - [options.packages.find] @@ -84,6 +82,9 @@ full = extractcode_libarchive >= 3.5.1.210525 typecode[full] >= 21.6.1 +patch = + patch >= 1.16 + testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 diff --git a/src/extractcode/archive.py b/src/extractcode/archive.py index 81135b3..d52399e 100644 --- a/src/extractcode/archive.py +++ b/src/extractcode/archive.py @@ -1194,5 +1194,11 @@ def try_to_extract(location, target_dir, extractor): QCOWHandler, VMDKHandler, VirtualBoxHandler, - PatchHandler, ] + +# only support extracting patches if patch is installed. This is not a default +try: + import patch as _pythonpatch + archive_handlers.append(PatchHandler) +except: + pass diff --git a/src/extractcode/patch.py b/src/extractcode/patch.py index 0225e45..3588695 100644 --- a/src/extractcode/patch.py +++ b/src/extractcode/patch.py @@ -11,8 +11,6 @@ import logging import os.path -import patch as pythonpatch - from commoncode import paths from commoncode import fileutils from commoncode import text @@ -136,6 +134,7 @@ def patch_info(location): Raise an exception if the file is not a patch file or cannot be parsed. """ + import patch as pythonpatch patchset = pythonpatch.fromfile(location) if not patchset: msg = 'Unable to parse patch file: %(location)s' % locals() diff --git a/tests/test_patch.py b/tests/test_patch.py index 5a70951..c3f4008 100644 --- a/tests/test_patch.py +++ b/tests/test_patch.py @@ -17,6 +17,12 @@ from extractcode import patch +try: + import patch as _pythonpatch +except ImportError: + import pytest + pytestmark = pytest.mark.skipif(True, reason="Run only if patch is installed.") + class TestIsPatch(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') From 99d274dac7655ed7eca198e02a8e0e5ba805cbef Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 9 May 2022 12:57:18 +0200 Subject: [PATCH 60/62] Update requirements to latest * Sync up with ScanCode TK. * Remove unused files Signed-off-by: Philippe Ombredanne --- .travis.yml | 22 ----- CODE_OF_CONDUCT.rst | 86 ++++++++++++++++++ MANIFEST.in | 4 +- README.rst | 5 +- appveyor.yml | 19 ++++ azure-pipelines.yml | 8 -- configure | 5 +- docs/source/conf.py | 6 +- docs/source/skeleton-usage.rst | 160 --------------------------------- requirements-dev.txt | 29 ++++-- requirements.txt | 89 +++++++++++++----- setup.cfg | 10 ++- 12 files changed, 211 insertions(+), 232 deletions(-) delete mode 100644 .travis.yml create mode 100644 CODE_OF_CONDUCT.rst create mode 100644 appveyor.yml delete mode 100644 docs/source/skeleton-usage.rst diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b022fc4..0000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -# This is a skeleton Travis CI config file that provides a starting point for adding CI -# to a Python project. Since we primarily develop in python3, this skeleton config file -# will be specific to that language. -# -# See https://config.travis-ci.com/ for a full list of configuration options. - -os: linux - -dist: xenial - -language: python -python: - - "3.6" - - "3.7" - - "3.8" - - "3.9" - -# Scripts to run at install stage -install: ./configure --dev - -# Scripts to run at script stage -script: venv/bin/pytest --ignore=tests/test_vmimage.py diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst new file mode 100644 index 0000000..590ba19 --- /dev/null +++ b/CODE_OF_CONDUCT.rst @@ -0,0 +1,86 @@ +Contributor Covenant Code of Conduct +==================================== + +Our Pledge +---------- + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our +project and our community a harassment-free experience for everyone, +regardless of age, body size, disability, ethnicity, gender identity and +expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +Our Standards +------------- + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual + attention or advances +- Trolling, insulting/derogatory comments, and personal or political + attacks +- Public or private harassment +- Publishing others’ private information, such as a physical or + electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +Our Responsibilities +-------------------- + +Project maintainers are responsible for clarifying the standards of +acceptable behavior and are expected to take appropriate and fair +corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, +or reject comments, commits, code, wiki edits, issues, and other +contributions that are not aligned to this Code of Conduct, or to ban +temporarily or permanently any contributor for other behaviors that they +deem inappropriate, threatening, offensive, or harmful. + +Scope +----- + +This Code of Conduct applies both within project spaces and in public +spaces when an individual is representing the project or its community. +Examples of representing a project or community include using an +official project e-mail address, posting via an official social media +account, or acting as an appointed representative at an online or +offline event. Representation of a project may be further defined and +clarified by project maintainers. + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may +be reported by contacting the project team at pombredanne@gmail.com +or on the Gitter chat channel at https://gitter.im/aboutcode-org/discuss . +All complaints will be reviewed and investigated and will result in a +response that is deemed necessary and appropriate to the circumstances. +The project team is obligated to maintain confidentiality with regard to +the reporter of an incident. Further details of specific enforcement +policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in +good faith may face temporary or permanent repercussions as determined +by other members of the project’s leadership. + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant`_ , +version 1.4, available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +.. _Contributor Covenant: https://www.contributor-covenant.org diff --git a/MANIFEST.in b/MANIFEST.in index ef3721e..c17471b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,7 +9,9 @@ include *.rst include setup.* include configure* include requirements* -include .git* +include .giti* + +include extractcode* global-exclude *.py[co] __pycache__ *.*~ diff --git a/README.rst b/README.rst index 94e6141..24063b8 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,6 @@ +============ ExtractCode -=========== +============ - license: Apache-2.0 - copyright: copyright (c) nexB. Inc. and others @@ -21,7 +22,7 @@ Why another extractor? **it will extract!** -ExtractCode will extract things where other extractors may fail. +ExtractCode will extract things where other archive and compressed file extractors may fail. ExtractCode supports one of largest number of archive formats listed in the long `List of supported archive formats`_ found at the bottom of this document. diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..6c9d089 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,19 @@ + +################################################################################ +# We use Appveyor to run minimal smoke tests suites on Pythons 3.x +# on Windows 64 bits +################################################################################ +environment: + matrix: + - PYTHON: "C:\\Python36-x64" +# - PYTHON: "C:\\Python37-x64" +# - PYTHON: "C:\\Python38-x64" +# - PYTHON: "C:\\Python39-x64" + + +build: off + + +test_script: + - python -c "import sys;print(sys.getdefaultencoding())" + - cmd: "set PYTHON_EXECUTABLE=%PYTHON%\\python.exe && configure --dev && venv\\Scripts\\pytest -vvs tests" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cc858bd..c0b2c58 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,14 +7,6 @@ jobs: - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu16_cpython - image_name: ubuntu-16.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] - test_suites: - all: sudo chmod 0644 /boot/vmlinuz-* && sudo apt install libguestfs-tools && tmp/bin/pytest -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu18_cpython diff --git a/configure b/configure index 319dffe..397f0e4 100755 --- a/configure +++ b/configure @@ -60,9 +60,8 @@ if [ -d "$CFG_ROOT_DIR/thirdparty" ]; then fi PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" - ################################ -# Set the quiet flag to empty if not defined +# Set the quiet flag to empty if not defined if [[ "$CFG_QUIET" == "" ]]; then CFG_QUIET=" " fi @@ -180,7 +179,7 @@ while getopts :-: optchar; do dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; -esac + esac done PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" diff --git a/docs/source/conf.py b/docs/source/conf.py index 62bca04..112fa55 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,7 +17,7 @@ # -- Project information ----------------------------------------------------- -project = "nexb-skeleton" +project = "ExtractCode" copyright = "nexB Inc. and others." author = "AboutCode.org authors and contributors" @@ -67,8 +67,8 @@ html_context = { "display_github": True, "github_user": "nexB", - "github_repo": "nexb-skeleton", - "github_version": "develop", # branch + "github_repo": "extractcode", + "github_version": "main", # branch "conf_py_path": "/docs/source/", # path in the checkout to the docs root } diff --git a/docs/source/skeleton-usage.rst b/docs/source/skeleton-usage.rst deleted file mode 100644 index cde23dc..0000000 --- a/docs/source/skeleton-usage.rst +++ /dev/null @@ -1,160 +0,0 @@ -Usage -===== -A brand new project -------------------- -.. code-block:: bash - - git init my-new-repo - cd my-new-repo - git pull git@github.com:nexB/skeleton - - # Create the new repo on GitHub, then update your remote - git remote set-url origin git@github.com:nexB/your-new-repo.git - -From here, you can make the appropriate changes to the files for your specific project. - -Update an existing project ---------------------------- -.. code-block:: bash - - cd my-existing-project - git remote add skeleton git@github.com:nexB/skeleton - git fetch skeleton - git merge skeleton/main --allow-unrelated-histories - -This is also the workflow to use when updating the skeleton files in any given repository. - -Customizing ------------ - -You typically want to perform these customizations: - -- remove or update the src/README.rst and tests/README.rst files -- set project info and dependencies in setup.cfg -- check the configure and configure.bat defaults - -Initializing a project ----------------------- - -All projects using the skeleton will be expected to pull all of it dependencies -from thirdparty.aboutcode.org/pypi or the local thirdparty directory, using -requirements.txt and/or requirements-dev.txt to determine what version of a -package to collect. By default, PyPI will not be used to find and collect -packages from. - -In the case where we are starting a new project where we do not have -requirements.txt and requirements-dev.txt and whose dependencies are not yet on -thirdparty.aboutcode.org/pypi, we run the following command after adding and -customizing the skeleton files to your project: - -.. code-block:: bash - - ./configure - -This will initialize the virtual environment for the project, pull in the -dependencies from PyPI and add them to the virtual environment. - - -Generating requirements.txt and requirements-dev.txt ----------------------------------------------------- - -After the project has been initialized, we can generate the requirements.txt and -requirements-dev.txt files. - -Ensure the virtual environment is enabled. - -.. code-block:: bash - - source venv/bin/activate - -To generate requirements.txt: - -.. code-block:: bash - - python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ - -Replace \ with the version number of the Python being used, for example: -``venv/lib/python3.6/site-packages/`` - -To generate requirements-dev.txt after requirements.txt has been generated: - -.. code-block:: bash - - ./configure --dev - python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ - -Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` - -.. code-block:: bash - - python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ - .\configure --dev - python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ - - -Collecting and generating ABOUT files for dependencies ------------------------------------------------------- - -Ensure that the dependencies used by ``etc/scripts/fetch_thirdparty.py`` are installed: - -.. code-block:: bash - - pip install -r etc/scripts/requirements.txt - -Once we have requirements.txt and requirements-dev.txt, we can fetch the project -dependencies as wheels and generate ABOUT files for them: - -.. code-block:: bash - - python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt - -There may be issues with the generated ABOUT files, which will have to be -corrected. You can check to see if your corrections are valid by running: - -.. code-block:: bash - - python etc/scripts/check_thirdparty.py -d thirdparty - -Once the wheels are collected and the ABOUT files are generated and correct, -upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT -files from the thirdparty directory to the pypi directory at -https://github.com/nexB/thirdparty-packages - - -Usage after project initialization ----------------------------------- - -Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated -and the project dependencies and their ABOUT files have been uploaded to -thirdparty.aboutcode.org/pypi, you can configure the project as needed, typically -when you update dependencies or use a new checkout. - -If the virtual env for the project becomes polluted, or you would like to remove -it, use the ``--clean`` option: - -.. code-block:: bash - - ./configure --clean - -Then you can run ``./configure`` again to set up the project virtual environment. - -To set up the project for development use: - -.. code-block:: bash - - ./configure --dev - -To update the project dependencies (adding, removing, updating packages, etc.), -update the dependencies in ``setup.cfg``, then run: - -.. code-block:: bash - - ./configure --clean # Remove existing virtual environment - source venv/bin/activate # Ensure virtual environment is activated - python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt - python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt - pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py - python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt # Collect dependency wheels and their ABOUT files - -Ensure that the generated ABOUT files are valid, then take the dependency wheels -and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. diff --git a/requirements-dev.txt b/requirements-dev.txt index 4dcff74..fe92ed8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,24 @@ +aboutcode-toolkit==7.0.1 +bleach==4.1.0 +build==0.7.0 +commonmark==0.9.1 +docutils==0.18.1 +et-xmlfile==1.1.0 execnet==1.9.0 iniconfig==1.1.1 -packaging==21.0 -py==1.10.0 -pyparsing==2.4.7 -pytest==6.2.5 -pytest-forked==1.3.0 -pytest-xdist==2.4.0 -toml==0.10.2 \ No newline at end of file +jeepney==0.7.1 +keyring==23.4.1 +openpyxl==3.0.9 +pep517==0.12.0 +pkginfo==1.8.2 +py==1.11.0 +pytest==7.0.1 +pytest-forked==1.4.0 +pytest-xdist==2.5.0 +readme-renderer==34.0 +requests-toolbelt==0.9.1 +rfc3986==1.5.0 +rich==12.3.0 +secretstorage==3.3.2 +tomli==1.2.3 +twine==3.8.0 diff --git a/requirements.txt b/requirements.txt index c5d3aad..627778d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,36 +1,79 @@ -attrs==21.2.0 -beautifulsoup4==4.10.0 +attrs==21.4.0 +banal==1.0.6 +beautifulsoup4==4.11.1 binaryornot==0.4.4 -certifi==2021.5.30 -cffi==1.14.6 +boolean.py==3.8 +certifi==2021.10.8 +cffi==1.15.0 chardet==4.0.0 -charset-normalizer==2.0.6 -click==8.0.1 -commoncode==30.0.0 -cryptography==35.0.0 +charset-normalizer==2.0.12 +click==8.0.4 +colorama==0.4.4 +commoncode==30.2.0 +construct==2.10.68 +container-inspector==30.0.0 +cryptography==36.0.2 +debian-inspector==30.0.0 +dockerfile-parse==1.2.0 +dparse2==0.6.1 extractcode-7z==16.5.210531 extractcode-libarchive==3.5.1.210531 -idna==3.2 -importlib-metadata==4.8.1 -intbitset==2.4.1 +fasteners==0.17.3 +fingerprints==1.0.3 +ftfy==6.0.3 +future==0.18.2 +gemfileparser==0.8.0 +html5lib==1.1 +idna==3.3 +importlib-metadata==4.8.3 +inflection==0.5.1 +intbitset==3.0.1 +isodate==0.6.1 +jaraco.functools==3.4.0 +javaproperties==0.8.1 +Jinja2==3.0.3 +jsonstreams==0.6.0 +license-expression==21.6.14 +lxml==4.8.0 +MarkupSafe==2.0.1 +more-itertools==8.13.0 +normality==2.3.3 +packagedcode-msitools==0.101.210706 +packageurl-python==0.9.9 +packaging==21.3 +parameter-expansion-patched==0.3.1 patch==1.16 -pdfminer.six==20201018 -pip==21.2.4 +pdfminer.six==20220506 +pefile==2021.9.3 +pip-requirements-parser==31.2.0 +pkginfo2==30.0.0 pluggy==1.0.0 plugincode==21.1.21 -pycparser==2.20 -PyYAML==5.4.1 -requests==2.26.0 +ply==3.11 +publicsuffix2==2.20191221 +pyahocorasick==2.0.0b1 +pycparser==2.21 +pygmars==0.7.0 +Pygments==2.12.0 +pymaven-patch==0.3.0 +pyparsing==3.0.8 +pytz==2022.1 +PyYAML==6.0 +rdflib==5.0.0 +regipy==2.2.2 +requests==2.27.1 +rpm-inspector-rpm==4.16.1.3.210404 saneyaml==0.5.2 -setuptools==58.1.0 six==1.16.0 -sortedcontainers==2.4.0 -soupsieve==2.2.1 +soupsieve==2.3.1 +spdx-tools==0.7.0a3 text-unidecode==1.3 +toml==0.10.2 typecode==21.6.1 typecode-libmagic==5.39.210531 -typing==3.6.6 -typing-extensions==3.10.0.2 -urllib3==1.26.7 -wheel==0.37.0 +urllib3==1.26.9 +urlpy==0.5 +wcwidth==0.2.5 +webencodings==0.5.1 +xmltodict==0.12.0 zipp==3.6.0 diff --git a/setup.cfg b/setup.cfg index 1515c28..0107261 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,11 @@ classifiers = Intended Audience :: Developers Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Topic :: Software Development Topic :: Utilities @@ -53,6 +58,7 @@ license_files = NOTICE AUTHORS.rst CHANGELOG.rst + CODE_OF_CONDUCT.rst [options] package_dir = @@ -61,13 +67,11 @@ packages = find: include_package_data = true zip_safe = false -setup_requires = setuptools_scm[toml] >= 4 - python_requires = >=3.6.* install_requires = attrs >= 18.1, !=20.1.0 - commoncode >= 21.5.25 + commoncode >= 30.2.0 plugincode >= 21.1.21 typecode >= 21.6.1 From a6cd72b6e744fb66ec0a2e6a8d9e2cbed95cb253 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 9 May 2022 14:48:19 +0200 Subject: [PATCH 61/62] Install required six Signed-off-by: Philippe Ombredanne --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0107261..e4eeb8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -74,7 +74,7 @@ install_requires = commoncode >= 30.2.0 plugincode >= 21.1.21 typecode >= 21.6.1 - + six [options.packages.find] where = src From 56c887bf76c526c8fe1d9ba35789389b68d16dbf Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Mon, 9 May 2022 17:25:38 +0200 Subject: [PATCH 62/62] Remove unused doc reference Signed-off-by: Philippe Ombredanne --- docs/source/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index eb63717..68099e7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -5,7 +5,6 @@ Welcome to nexb-skeleton's documentation! :maxdepth: 2 :caption: Contents: - skeleton-usage contribute/contrib_doc Indices and tables