diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ddcede2c..2125a3af 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,6 +34,12 @@ jobs: - python-version: '3.10' os: ubuntu-latest tox-env: py + - python-version: '3.11' + os: ubuntu-latest + tox-env: py + - python-version: '3.12' + os: ubuntu-latest + tox-env: py steps: - uses: actions/checkout@v2 @@ -74,6 +80,10 @@ jobs: tox-env: py - python-version: '3.10' tox-env: py + - python-version: '3.11' + tox-env: py + - python-version: '3.12' + tox-env: py steps: - uses: actions/checkout@v2 @@ -108,6 +118,10 @@ jobs: tox-env: py - python-version: '3.10' tox-env: py + - python-version: '3.11' + tox-env: py + - python-version: '3.12' + tox-env: py steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 017290bc..afc2f1f1 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,8 @@ install_requires=[ 'click', 'docker', + 'importlib-metadata; python_version < "3.8"', + 'packaging', 'pip', 'PyYAML', 'retrying', @@ -52,6 +54,8 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Operating System :: OS Independent', 'Environment :: Console', 'Topic :: Internet :: WWW/HTTP', diff --git a/shub/deploy_egg.py b/shub/deploy_egg.py index 4f514ca5..5f281682 100644 --- a/shub/deploy_egg.py +++ b/shub/deploy_egg.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import os import tempfile +from shutil import which import click @@ -9,7 +10,7 @@ from shub.exceptions import (BadParameterException, NotFoundException, SubcommandException) from shub.utils import (decompress_egg_files, download_from_pypi, - find_executable, run_cmd) + run_cmd) HELP = """ @@ -86,7 +87,7 @@ def _checkout(repo, git_branch=None, target_dir='egg-tmp-clone'): ] missing_exes = [] for cmd in vcs_commands: - exe = find_executable(cmd[0]) + exe = which(cmd[0]) if not exe: missing_exes.append(cmd[0]) continue @@ -109,7 +110,7 @@ def _checkout(repo, git_branch=None, target_dir='egg-tmp-clone'): if git_branch: try: - run_cmd([find_executable('git'), 'checkout', git_branch]) + run_cmd([which('git'), 'checkout', git_branch]) except SubcommandException: raise BadParameterException("Branch %s is not valid" % git_branch) click.echo("%s branch was checked out" % git_branch) diff --git a/shub/image/run/wrapper.py b/shub/image/run/wrapper.py index bb39d98f..ec0472a2 100644 --- a/shub/image/run/wrapper.py +++ b/shub/image/run/wrapper.py @@ -29,7 +29,7 @@ import logging import datetime from multiprocessing import Process -from distutils.spawn import find_executable +from shutil import which def _consume_from_fifo(fifo_path): @@ -68,7 +68,7 @@ def main(): # non-daemon to allow it to finish reading from pipe before exit. Process(target=_consume_from_fifo, args=[fifo_path]).start() # replace current process with original start-crawl - os.execv(find_executable('start-crawl'), sys.argv) + os.execv(which('start-crawl'), sys.argv) if __name__ == '__main__': diff --git a/shub/image/utils.py b/shub/image/utils.py index 447e9351..18aeb696 100644 --- a/shub/image/utils.py +++ b/shub/image/utils.py @@ -9,7 +9,6 @@ import yaml from tqdm import tqdm from six import binary_type -import pkg_resources from shub import config as shub_config from shub import utils as shub_utils @@ -18,6 +17,11 @@ ShubDeprecationWarning, print_warning, BadParameterException, ) +if sys.version_info < (3, 8): + import importlib_metadata as metadata +else: + from importlib import metadata + DEFAULT_DOCKER_API_VERSION = '1.21' STATUS_FILE_LOCATION = '.releases' @@ -83,8 +87,8 @@ def get_docker_client(validate=True): import docker except ImportError: raise ImportError(DOCKER_PY_UNAVAILABLE_MSG) - for dep in pkg_resources.working_set: - if dep.project_name == 'docker-py': + for dep in metadata.distributions(): + if dep.name == 'docker-py': raise ImportError(DOCKER_PY_UNAVAILABLE_MSG) docker_host = os.environ.get('DOCKER_HOST') diff --git a/shub/utils.py b/shub/utils.py index ae25fd6e..14d9eb5d 100644 --- a/shub/utils.py +++ b/shub/utils.py @@ -11,9 +11,9 @@ import time from collections import deque -from six.moves.configparser import SafeConfigParser -from distutils.spawn import find_executable -from distutils.version import LooseVersion, StrictVersion +from configparser import ConfigParser +from shutil import which +from packaging.version import Version from glob import glob from importlib import import_module from tempfile import NamedTemporaryFile, TemporaryFile @@ -143,17 +143,21 @@ def _check_deploy_files_size(files): def write_and_echo_logs(keep_log, last_logs, rsp, verbose): """It will write logs to temporal file and echo if verbose is True.""" - with NamedTemporaryFile(prefix='shub_deploy_', suffix='.log', - delete=(not keep_log)) as log_file: - for line in rsp.iter_lines(): - if verbose: - click.echo(line) - last_logs.append(line) - log_file.write(line + b'\n') + log_contents = b"" + for line in rsp.iter_lines(): + if verbose: + click.echo(line) + last_logs.append(line) + log_contents += line + b'\n' + deployed = _is_deploy_successful(last_logs) + if not deployed: + keep_log = True + echo_short_log_if_deployed(deployed, last_logs, verbose=verbose) - deployed = _is_deploy_successful(last_logs) - echo_short_log_if_deployed(deployed, last_logs, log_file, verbose) - if not log_file.delete: + with NamedTemporaryFile(prefix='shub_deploy_', suffix='.log', + delete=not keep_log) as log_file: + log_file.write(log_contents) + if keep_log: click.echo("Deploy log location: %s" % log_file.name) if not deployed: try: @@ -163,12 +167,11 @@ def write_and_echo_logs(keep_log, last_logs, rsp, verbose): raise RemoteErrorException("Deploy failed: {}".format(last_log)) -def echo_short_log_if_deployed(deployed, last_logs, log_file, verbose): +def echo_short_log_if_deployed(deployed, last_logs, log_file=None, verbose=False): if deployed: if not verbose: click.echo(last_logs[-1]) else: - log_file.delete = False if not verbose: click.echo("Deploy log last %s lines:" % len(last_logs)) for line in last_logs: @@ -212,7 +215,7 @@ def patch_sys_executable(): def find_exe(exe_name): - exe = find_executable(exe_name) + exe = which(exe_name) if not exe: raise NotFoundException("Please install {}".format(exe_name)) return exe @@ -275,7 +278,7 @@ def pwd_version(): def pwd_git_version(): - git = find_executable('git') + git = which('git') if not git: return None try: @@ -290,7 +293,7 @@ def pwd_git_version(): def pwd_hg_version(): - hg = find_executable('hg') + hg = which('hg') if not hg: return None try: @@ -302,7 +305,7 @@ def pwd_hg_version(): def pwd_bzr_version(): - bzr = find_executable('bzr') + bzr = which('bzr') if not bzr: return None try: @@ -485,9 +488,9 @@ def inside_project(): def get_config(use_closest=True): - """Get Scrapy config file as a SafeConfigParser""" + """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) - cfg = SafeConfigParser() + cfg = ConfigParser() cfg.read(sources) return cfg @@ -506,7 +509,7 @@ def get_sources(use_closest=True): def get_scrapycfg_targets(cfgfiles=None): - cfg = SafeConfigParser() + cfg = ConfigParser() cfg.read(cfgfiles or []) baset = dict(cfg.items('deploy')) if cfg.has_section('deploy') else {} targets = {} @@ -627,8 +630,8 @@ def update_available(silent_fail=True): """ try: release_data = latest_github_release() - latest_rls = StrictVersion(release_data['name'].lstrip('v')) - used_rls = StrictVersion(shub.__version__) + latest_rls = Version(release_data['name'].lstrip('v')) + used_rls = Version(shub.__version__) if used_rls >= latest_rls: return None return release_data['html_url'] @@ -643,15 +646,15 @@ def download_from_pypi(dest, pkg=None, reqfile=None, extra_args=None): if (not pkg and not reqfile) or (pkg and reqfile): raise ValueError('Call with either pkg or reqfile') extra_args = extra_args or [] - pip_version = LooseVersion(getattr(pip, '__version__', '1.0')) + pip_version = Version(getattr(pip, '__version__', '1.0')) cmd = 'install' no_wheel = [] target = [pkg] if pkg else ['-r', reqfile] - if pip_version >= LooseVersion('1.4'): + if pip_version >= Version('1.4'): no_wheel = ['--no-use-wheel'] - if pip_version >= LooseVersion('7'): + if pip_version >= Version('7'): no_wheel = ['--no-binary=:all:'] - if pip_version >= LooseVersion('8'): + if pip_version >= Version('8'): cmd = 'download' with patch_sys_executable(): pip_main([cmd, '-d', dest, '--no-deps'] + no_wheel + extra_args + diff --git a/tests/test_config.py b/tests/test_config.py index f39a4038..0b69e525 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -126,7 +126,7 @@ def test_load(self): } self.assertEqual(projects, self.conf.projects) endpoints = {'external': 'ext_endpoint'} - self.assertDictContainsSubset(endpoints, self.conf.endpoints) + self.assertLessEqual(endpoints.items(), self.conf.endpoints.items()) apikeys = {'default': 'key', 'otheruser': 'otherkey'} self.assertEqual(apikeys, self.conf.apikeys) stacks = {'dev': 'scrapy:v1.1'} @@ -145,7 +145,7 @@ def test_load_partial(self): """ conf = self._get_conf_with_yml(yml) endpoints = {'external': 'ext_endpoint'} - self.assertDictContainsSubset(endpoints, conf.endpoints) + self.assertLessEqual(endpoints.items(), conf.endpoints.items()) self.assertEqual(conf.projects, {}) self.assertEqual(conf.apikeys, {}) self.assertEqual(conf.images, {}) @@ -176,9 +176,9 @@ def test_load_shortcut_mixed(self): dev: dev_stack stack: prod_stack """ - self.assertDictContainsSubset( - self._get_conf_with_yml(yml).stacks, - {'default': 'prod_stack', 'dev': 'dev_stack'}, + self.assertLessEqual( + self._get_conf_with_yml(yml).stacks.items(), + {'default': 'prod_stack', 'dev': 'dev_stack'}.items() ) def test_load_shortcut_conflict(self): @@ -586,7 +586,7 @@ def test_get_image_ambiguous_global_image_and_global_stack(self): image: true stack: scrapy:1.3 """) - with self.assertRaisesRegexp(BadConfigException, '(?i)ambiguous'): + with self.assertRaisesRegex(BadConfigException, '(?i)ambiguous'): self.conf.get_image('default') def test_get_image_ambiguous_global_image_and_project_stack(self): @@ -601,9 +601,9 @@ def test_get_image_ambiguous_global_image_and_project_stack(self): stack: scrapy:1.3 image: true """) - with self.assertRaisesRegexp(BadConfigException, '(?i)ambiguous'): + with self.assertRaisesRegex(BadConfigException, '(?i)ambiguous'): self.conf.get_image('bad') - with self.assertRaisesRegexp(BadConfigException, '(?i)disabled'): + with self.assertRaisesRegex(BadConfigException, '(?i)disabled'): self.conf.get_image('good') def test_get_image_ambiguous_project_image_and_project_stack(self): @@ -614,7 +614,7 @@ def test_get_image_ambiguous_project_image_and_project_stack(self): image: true stack: scrapy:1.3 """) - with self.assertRaisesRegexp(BadConfigException, '(?i)ambiguous'): + with self.assertRaisesRegex(BadConfigException, '(?i)ambiguous'): self.conf.get_image('default') def test_get_target_conf(self): diff --git a/tests/test_schedule.py b/tests/test_schedule.py index 85b1deb2..60756a98 100644 --- a/tests/test_schedule.py +++ b/tests/test_schedule.py @@ -63,9 +63,9 @@ def test_forwards_args_and_settings(self, mock_client): "--argument ARGWITHEQUAL=val2=val2".split(' '), ) job_args = mock_proj.jobs.run.call_args[1]['job_args'] - self.assertDictContainsSubset( - {'ARG': 'val1', 'ARGWITHEQUAL': 'val2=val2'}, - job_args, + self.assertLessEqual( + {'ARG': 'val1', 'ARGWITHEQUAL': 'val2=val2'}.items(), + job_args.items(), ) job_settings = mock_proj.jobs.run.call_args[1]['job_settings'] self.assertEqual( @@ -116,9 +116,9 @@ def test_forwards_environment(self, mock_client): "testspider -e VAR1=VAL1 --environment VAR2=VAL2".split(' '), ) call_kwargs = mock_proj.jobs.run.call_args[1] - self.assertDictContainsSubset( - {'VAR1': 'VAL1', 'VAR2': 'VAL2'}, - call_kwargs['environment'], + self.assertLessEqual( + {'VAR1': 'VAL1', 'VAR2': 'VAL2'}.items(), + call_kwargs['environment'].items(), ) diff --git a/tests/test_utils.py b/tests/test_utils.py index a71f7512..8016afda 100755 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -11,6 +11,7 @@ import unittest import textwrap import time +from io import StringIO from unittest.mock import Mock, MagicMock, patch import click @@ -50,7 +51,7 @@ def test_patch_sys_executable(self, mock_find_exe): with utils.patch_sys_executable(): pass - @patch('shub.utils.find_executable') + @patch('shub.utils.which') def test_find_exe(self, mock_fe): mock_fe.return_value = '/usr/bin/python' self.assertEqual(utils.find_exe('python'), '/usr/bin/python') @@ -66,7 +67,7 @@ def test_run_cmd_captures_stderr(self): 'print("Hello", file=sys.stderr)', ] self.assertEqual(utils.run_cmd(cmd), '') - with self.assertRaisesRegexp(SubcommandException, r'STDERR[\s-]+Hello'): + with self.assertRaisesRegex(SubcommandException, r'STDERR[\s-]+Hello'): cmd[-1] += '; sys.exit(99)' utils.run_cmd(cmd) @@ -74,7 +75,7 @@ def test_pwd_git_version_without_git(self): # Change into test dir to make sure we're within a repo os.chdir(os.path.dirname(__file__)) self.assertIsNotNone(utils.pwd_git_version()) - with patch('shub.utils.find_executable', return_value=None): + with patch('shub.utils.which', return_value=None): self.assertIsNone(utils.pwd_git_version()) @patch('shub.utils.pwd_git_version', return_value='ver_GIT') @@ -274,46 +275,46 @@ def jri_result(follow, tail=None): def test_latest_github_release(self, mock_get): with self.runner.isolated_filesystem(): mock_get.return_value.json.return_value = {'key': 'value'} - self.assertDictContainsSubset( - {'key': 'value'}, - utils.latest_github_release(cache='./cache.txt'), + self.assertLessEqual( + {'key': 'value'}.items(), + utils.latest_github_release(cache='./cache.txt').items(), ) mock_get.return_value.json.return_value = {'key': 'newvalue'} - self.assertDictContainsSubset( - {'key': 'value'}, - utils.latest_github_release(cache='./cache.txt'), + self.assertLessEqual( + {'key': 'value'}.items(), + utils.latest_github_release(cache='./cache.txt').items(), ) - self.assertDictContainsSubset( - {'key': 'newvalue'}, + self.assertLessEqual( + {'key': 'newvalue'}.items(), utils.latest_github_release(force_update=True, - cache='./cache.txt'), + cache='./cache.txt').items(), ) # Garbage in cache mock_get.return_value.json.return_value = {'key': 'value'} with open('./cache.txt', 'w') as f: f.write('abc') - self.assertDictContainsSubset( - {'key': 'value'}, - utils.latest_github_release(cache='./cache.txt'), + self.assertLessEqual( + {'key': 'value'}.items(), + utils.latest_github_release(cache='./cache.txt').items(), ) mock_get.return_value.json.return_value = {'key': 'newvalue'} - self.assertDictContainsSubset( - {'key': 'value'}, - utils.latest_github_release(cache='./cache.txt'), + self.assertLessEqual( + {'key': 'value'}.items(), + utils.latest_github_release(cache='./cache.txt').items(), ) # Readonly cache file mock_get.return_value.json.return_value = {'key': 'value'} with open('./cache.txt', 'w') as f: f.write('abc') os.chmod('./cache.txt', stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - self.assertDictContainsSubset( - {'key': 'value'}, - utils.latest_github_release(cache='./cache.txt'), + self.assertLessEqual( + {'key': 'value'}.items(), + utils.latest_github_release(cache='./cache.txt').items(), ) mock_get.return_value.json.return_value = {'key': 'newvalue'} - self.assertDictContainsSubset( - {'key': 'newvalue'}, - utils.latest_github_release(cache='./cache.txt'), + self.assertLessEqual( + {'key': 'newvalue'}.items(), + utils.latest_github_release(cache='./cache.txt').items(), ) with open('./cache.txt', 'r') as f: self.assertEqual(f.read(), 'abc') @@ -382,20 +383,19 @@ def _call(*args, **kwargs): self.assertIn('--no-binary=:all:', pipargs) def test_echo_short_log_if_deployed(self): - log_file = Mock(delete=None) last_logs = ["last log line"] deployed = True - for verbose in [True, False]: - utils.echo_short_log_if_deployed( - deployed, last_logs, log_file, verbose) - self.assertEqual(None, log_file.delete) + for verbose, expected in zip([True, False], ["", "last log line\n"]): + with patch('sys.stdout', new_callable=StringIO) as stdout: + utils.echo_short_log_if_deployed(deployed, last_logs, verbose=verbose) + self.assertEqual(expected, stdout.getvalue()) deployed = False - for verbose in [True, False]: - utils.echo_short_log_if_deployed( - deployed, last_logs, log_file, verbose) - self.assertEqual(False, log_file.delete) + for verbose, expected in zip([True, False], ["", "Deploy log last 1 lines:\nlast log line\n"]): + with patch('sys.stdout', new_callable=StringIO) as stdout: + utils.echo_short_log_if_deployed(deployed, last_logs, verbose=verbose) + self.assertEqual(expected, stdout.getvalue()) def test_write_and_echo_logs(self): last_logs = []