From a061644fa8c748347f3795d9a726c53506c9f669 Mon Sep 17 00:00:00 2001 From: bpinsard Date: Thu, 8 Aug 2024 15:49:42 -0400 Subject: [PATCH 1/2] pre-commit fixes on all files to avoid other PR to contain unrelated changes --- .github/ISSUE_TEMPLATE/issue_template.yml | 2 +- .github/ISSUE_TEMPLATE/issue_template_bug.yml | 2 +- .zenodo.json | 2 +- README.md | 2 +- datalad/__init__.py | 1 + datalad/__main__.py | 3 +- datalad/_version.py | 7 +- datalad/api.py | 21 +++--- datalad/cli/__init__.py | 1 + datalad/cli/common_args.py | 6 +- datalad/cli/exec.py | 11 ++- datalad/cli/helpers.py | 16 +++-- datalad/cli/renderer.py | 5 +- datalad/cli/tests/test_exec.py | 1 - datalad/cli/tests/test_renderer.py | 3 - datalad/cli/tests/test_utils.py | 3 - datalad/cli/utils.py | 1 - datalad/cmdline/__init__.py | 1 + datalad/cmdline/helpers.py | 4 +- datalad/cmdline/main.py | 1 + datalad/config.py | 2 +- datalad/consts.py | 2 +- datalad/core/distributed/clone.py | 8 +-- datalad/core/distributed/clone_ria.py | 4 +- datalad/core/distributed/clone_utils.py | 3 +- datalad/core/distributed/push.py | 40 +++++------ datalad/core/local/create.py | 52 ++++++-------- datalad/core/local/diff.py | 34 ++++----- datalad/core/local/repo.py | 7 +- datalad/core/local/resulthooks.py | 2 +- datalad/core/local/status.py | 39 +++++----- datalad/coreapi.py | 6 +- datalad/customremotes/__init__.py | 9 ++- datalad/customremotes/base.py | 1 + datalad/customremotes/main.py | 4 +- datalad/customremotes/ria_utils.py | 1 - datalad/customremotes/tests/test_main.py | 8 +-- datalad/dataset/gitrepo.py | 5 +- datalad/dataset/repo.py | 3 +- datalad/distributed/create_sibling_ghlike.py | 1 - datalad/distributed/create_sibling_gin.py | 3 +- datalad/distributed/drop.py | 2 +- datalad/distributed/export_archive_ora.py | 24 +++---- datalad/distributed/export_to_figshare.py | 30 +++++--- datalad/distributed/ora_remote.py | 42 ++++++----- .../tests/test_create_sibling_gitlab.py | 2 + datalad/distributed/tests/test_drop.py | 1 - datalad/distribution/create_sibling.py | 2 +- datalad/distribution/create_sibling_github.py | 1 + datalad/distribution/create_test_dataset.py | 30 +++++--- datalad/distribution/dataset.py | 39 +++++----- datalad/distribution/get.py | 71 +++++++++---------- datalad/distribution/install.py | 37 +++++----- .../distribution/tests/test_create_sibling.py | 2 +- datalad/distribution/tests/test_dataset.py | 4 +- datalad/distribution/tests/test_get.py | 22 +++--- datalad/distribution/tests/test_install.py | 8 +-- datalad/distribution/tests/test_siblings.py | 4 +- datalad/distribution/uninstall.py | 31 ++++---- datalad/distribution/update.py | 22 +++--- datalad/distribution/utils.py | 13 ++-- datalad/dochelpers.py | 4 +- datalad/downloaders/__init__.py | 6 +- datalad/downloaders/base.py | 36 ++++++---- datalad/downloaders/configs/indi.cfg | 2 - datalad/downloaders/configs/loris.cfg | 1 - datalad/downloaders/configs/openfmri.cfg | 2 - datalad/downloaders/credentials.py | 15 ++-- datalad/downloaders/http.py | 54 ++++++++------ datalad/downloaders/providers.py | 44 +++++++----- datalad/downloaders/s3.py | 32 ++++----- datalad/interface/add_archive_content.py | 1 + datalad/interface/base.py | 5 +- datalad/interface/clean.py | 1 + datalad/interface/common_cfg.py | 9 +-- datalad/interface/common_opts.py | 6 +- datalad/interface/download_url.py | 1 + datalad/interface/rerun.py | 1 + datalad/interface/run_procedure.py | 1 + datalad/interface/shell_completion.py | 4 +- datalad/interface/unlock.py | 1 + datalad/interface/utils.py | 40 +++++------ datalad/local/add_readme.py | 18 +++-- datalad/local/addurls.py | 2 +- datalad/local/check_dates.py | 15 ++-- datalad/local/copy_file.py | 33 ++++----- datalad/local/export_archive.py | 21 +++--- datalad/local/foreach_dataset.py | 13 ++-- datalad/local/gitcredential.py | 4 +- datalad/local/gitcredential_datalad.py | 2 +- datalad/local/no_annex.py | 23 +++--- datalad/local/remove.py | 4 +- datalad/local/rerun.py | 3 +- datalad/local/subdatasets.py | 43 ++++++----- datalad/local/tests/test_rerun.py | 2 +- datalad/local/tests/test_wtf.py | 3 +- datalad/local/wtf.py | 48 +++++++------ datalad/log.py | 21 +++--- datalad/resources/procedures/cfg_text2git.py | 2 +- datalad/resources/procedures/cfg_yoda.py | 2 +- .../procedures/tests/test_noannex.py | 2 +- datalad/runner/gitrunner.py | 3 +- datalad/runner/nonasyncrunner.py | 1 - datalad/runner/runner.py | 7 +- datalad/support/annexrepo.py | 1 - datalad/support/archive_utils_7z.py | 10 +-- datalad/support/archive_utils_patool.py | 44 +++++++----- datalad/support/cache.py | 1 - datalad/support/due.py | 8 ++- datalad/support/due_utils.py | 21 ++++-- datalad/support/extensions.py | 5 +- datalad/support/gitrepo.py | 8 +-- datalad/support/json_py.py | 11 ++- datalad/support/keyring_.py | 7 +- datalad/support/locking.py | 11 +-- datalad/support/nda_.py | 5 +- datalad/support/network.py | 2 +- datalad/support/parallel.py | 14 ++-- datalad/support/param.py | 3 +- datalad/support/repodates.py | 1 - datalad/support/s3.py | 11 +-- datalad/support/sshconnector.py | 56 ++++++++------- datalad/support/sshrun.py | 9 +-- datalad/support/tests/test_annexrepo.py | 1 - datalad/support/tests/test_extensions.py | 2 - .../support/tests/test_external_versions.py | 8 ++- datalad/support/tests/test_fileinfo.py | 2 +- datalad/support/tests/test_locking.py | 2 +- datalad/support/tests/test_network.py | 4 +- datalad/support/tests/test_path.py | 8 +-- datalad/support/tests/test_sshrun.py | 8 ++- .../support/third/loris_token_generator.py | 10 +-- .../support/third/nda_aws_token_generator.py | 3 +- datalad/support/vcr_.py | 15 ++-- datalad/tests/__init__.py | 1 - datalad/tests/ca/README | 2 +- datalad/tests/test_base.py | 5 +- datalad/tests/test_cmd.py | 2 +- datalad/tests/test_tests_utils_pytest.py | 6 +- datalad/tests/utils_pytest.py | 7 +- datalad/tests/utils_testdatasets.py | 4 +- datalad/tests/utils_testrepos.py | 4 +- datalad/ui/__init__.py | 13 ++-- datalad/ui/base.py | 5 +- datalad/ui/dialog.py | 14 ++-- datalad/ui/progressbars.py | 4 +- datalad/ui/utils.py | 8 ++- docs/casts/basic_search.sh | 2 +- docs/casts/reproducible_analysis.sh | 2 +- docs/design.rst | 6 +- docs/source/_extras/readme.txt | 2 +- docs/source/basics.rst | 4 +- docs/source/basics_cmdline.rst.in | 14 ++-- docs/source/basics_nesteddatasets.rst.in | 10 +-- docs/source/design/cli.rst | 1 - docs/source/design/credentials.rst | 2 +- docs/source/design/drop.rst | 2 +- docs/source/design/pos_vs_kw_parameters.rst | 2 +- docs/source/design/progress_reporting.rst | 2 +- docs/source/design/python_imports.rst | 15 ++-- docs/source/design/standard_parameters.rst | 2 +- docs/source/design/user_messaging.rst | 2 +- docs/source/glossary.rst | 2 +- docs/utils/pygments_ansi_color.py | 7 +- 164 files changed, 892 insertions(+), 803 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/issue_template.yml b/.github/ISSUE_TEMPLATE/issue_template.yml index 6dda17f0a4..33dd06e9d4 100644 --- a/.github/ISSUE_TEMPLATE/issue_template.yml +++ b/.github/ISSUE_TEMPLATE/issue_template.yml @@ -5,4 +5,4 @@ description: Report a generic issue which is not necessarily a bug body: - type: textarea attributes: - label: Description \ No newline at end of file + label: Description diff --git a/.github/ISSUE_TEMPLATE/issue_template_bug.yml b/.github/ISSUE_TEMPLATE/issue_template_bug.yml index 1eadc3e50a..f575889ff2 100644 --- a/.github/ISSUE_TEMPLATE/issue_template_bug.yml +++ b/.github/ISSUE_TEMPLATE/issue_template_bug.yml @@ -18,7 +18,7 @@ body: attributes: label: DataLad information description: | - What version of DataLad and git-annex do you use (run `datalad --version`)? + What version of DataLad and git-annex do you use (run `datalad --version`)? On what operating system (consider running `datalad wtf`)? diff --git a/.zenodo.json b/.zenodo.json index 60c4744fdb..eefceb69eb 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -157,4 +157,4 @@ "access_right": "open", "license": "MIT", "upload_type": "software" -} \ No newline at end of file +} diff --git a/README.md b/README.md index db49413732..ad26092127 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ # 10000-ft. overview -DataLad's purpose is to make data management and data distribution more accessible. +DataLad's purpose is to make data management and data distribution more accessible. To do so, it stands on the shoulders of [Git] and [Git-annex] to deliver a decentralized system for data exchange. This includes automated ingestion of data from online portals and exposing it in readily usable form as Git(-annex) diff --git a/datalad/__init__.py b/datalad/__init__.py index a8dd881958..76ef964f93 100644 --- a/datalad/__init__.py +++ b/datalad/__init__.py @@ -109,6 +109,7 @@ def enable_librarymode(): ConfigManager, warn_on_undefined_git_identity, ) + cfg = ConfigManager() warn_on_undefined_git_identity(cfg) diff --git a/datalad/__main__.py b/datalad/__main__.py index 87f34cb542..ba6af1b1da 100644 --- a/datalad/__main__.py +++ b/datalad/__main__.py @@ -9,6 +9,7 @@ """Helper to use datalad as a "runnable" module with -m datalad""" import sys + from . import __version__ from .log import lgr @@ -44,8 +45,8 @@ def runctx(cmd, globals=None, locals=None): def main(argv=None): - import os import getopt + import os if argv is None: argv = sys.argv diff --git a/datalad/_version.py b/datalad/_version.py index 63f0754063..7f823a7888 100644 --- a/datalad/_version.py +++ b/datalad/_version.py @@ -11,12 +11,15 @@ """Git implementation of _version.py.""" import errno +import functools import os import re import subprocess import sys -from typing import Callable, Dict -import functools +from typing import ( + Callable, + Dict, +) def get_keywords(): diff --git a/datalad/api.py b/datalad/api.py index 7d1dd36754..5c009c84a0 100644 --- a/datalad/api.py +++ b/datalad/api.py @@ -15,12 +15,15 @@ def _command_summary(): # Import here to avoid polluting the datalad.api namespace. from collections import defaultdict - from datalad.interface.base import alter_interface_docs_for_api - from datalad.interface.base import get_api_name - from datalad.interface.base import get_cmd_doc - from datalad.interface.base import get_cmd_summaries - from datalad.interface.base import get_interface_groups - from datalad.interface.base import load_interface + + from datalad.interface.base import ( + alter_interface_docs_for_api, + get_api_name, + get_cmd_doc, + get_cmd_summaries, + get_interface_groups, + load_interface, + ) groups = get_interface_groups() grp_short_descriptions = defaultdict(list) @@ -43,13 +46,13 @@ def _command_summary(): def _generate_extension_api(): """Auto detect all available extensions and generate an API from them """ - from datalad.support.entrypoints import iter_entrypoints + import logging + from datalad.interface.base import ( get_api_name, load_interface, ) - - import logging + from datalad.support.entrypoints import iter_entrypoints lgr = logging.getLogger('datalad.api') for ename, _, (grp_descr, interfaces) in iter_entrypoints( diff --git a/datalad/cli/__init__.py b/datalad/cli/__init__.py index ee14e680b1..3ff3034f90 100644 --- a/datalad/cli/__init__.py +++ b/datalad/cli/__init__.py @@ -8,4 +8,5 @@ __docformat__ = 'restructuredtext' import logging + lgr = logging.getLogger('datalad.cli') diff --git a/datalad/cli/common_args.py b/datalad/cli/common_args.py index eeae343620..c4ee2b2b4f 100644 --- a/datalad/cli/common_args.py +++ b/datalad/cli/common_args.py @@ -13,13 +13,13 @@ __all__ = ['common_args'] +from datalad.interface.base import eval_params +from datalad.utils import ensure_unicode + from .helpers import ( HelpAction, LogLevelAction, ) -from datalad.interface.base import eval_params -from datalad.utils import ensure_unicode - _log_level_names = ['critical', 'error', 'warning', 'info', 'debug'] diff --git a/datalad/cli/exec.py b/datalad/cli/exec.py index c966d3df69..d0925589bd 100644 --- a/datalad/cli/exec.py +++ b/datalad/cli/exec.py @@ -13,13 +13,8 @@ # caused unconditionally by .main, and .parser. from datalad import cfg - -from datalad.interface.base import ( - is_api_arg, -) -from datalad.utils import ( - getargspec, -) +from datalad.interface.base import is_api_arg +from datalad.utils import getargspec # only imported during command execution # .interface._has_eval_results_call @@ -50,6 +45,7 @@ def call_from_parser(cls, args): """ # XXX needs safety check for name collisions import inspect + from datalad.interface.base import _has_eval_results_call argspec = getargspec(cls.__call__, include_kwonlyargs=True) @@ -82,6 +78,7 @@ def call_from_parser(cls, args): kwargs['result_xfm'] = None if '{' in args.common_result_renderer: from .renderer import DefaultOutputRenderer + # stupid hack, could and should become more powerful kwargs['result_renderer'] = DefaultOutputRenderer( args.common_result_renderer) diff --git a/datalad/cli/helpers.py b/datalad/cli/helpers.py index 7530099921..8c8f847be1 100644 --- a/datalad/cli/helpers.py +++ b/datalad/cli/helpers.py @@ -12,25 +12,26 @@ __docformat__ = 'restructuredtext' import argparse +import gzip import os import re import sys -import gzip import textwrap from textwrap import wrap +from platformdirs import AppDirs + from datalad import __version__ # delay? from datalad.support.exceptions import CapturedException from datalad.ui.utils import get_console_width from datalad.utils import is_interactive -from platformdirs import AppDirs - dirs = AppDirs("datalad", "datalad.org") from logging import getLogger + lgr = getLogger('datalad.cli.helpers') @@ -88,9 +89,10 @@ def _get_long_help(self, parser): get_cmd_doc, load_interface, ) + from .interface import ( - get_cmdline_command_name, alter_interface_docs_for_cmdline, + get_cmdline_command_name, ) preamble = get_description_with_cmd_summary( # produce a mapping of command groups to @@ -153,6 +155,7 @@ def _get_all_interfaces(self): def _try_manpage(self, parser): try: import subprocess + # get the datalad manpage to use manfile = os.environ.get('MANPATH', '/usr/share/man') \ + '/man1/{0}.1.gz'.format(parser.prog.replace(' ', '-')) @@ -221,10 +224,9 @@ def _fix_datalad_ri(s): def get_description_with_cmd_summary(grp_short_descriptions, interface_groups, parser_description): - from .interface import ( - dedent_docstring, - ) from datalad.interface.base import get_cmd_summaries + + from .interface import dedent_docstring lgr.debug("Generating detailed description for the parser") console_width = get_console_width() diff --git a/datalad/cli/renderer.py b/datalad/cli/renderer.py index 8b2ebe6b47..a8b90c4c71 100644 --- a/datalad/cli/renderer.py +++ b/datalad/cli/renderer.py @@ -1,9 +1,8 @@ """Render results in a terminal""" -from collections import ( - defaultdict, -) import string +from collections import defaultdict + from datalad.ui import ui NA_STRING = 'N/A' # we might want to make it configurable via config diff --git a/datalad/cli/tests/test_exec.py b/datalad/cli/tests/test_exec.py index 542f3761d0..8174186026 100644 --- a/datalad/cli/tests/test_exec.py +++ b/datalad/cli/tests/test_exec.py @@ -174,4 +174,3 @@ def __call__(pos_arg, **kwargs): args = parser.parse_args(["val"]) eq_(call_from_parser(Cmd, args), "val") - diff --git a/datalad/cli/tests/test_renderer.py b/datalad/cli/tests/test_renderer.py index cc56f8e728..fa0c0e10df 100644 --- a/datalad/cli/tests/test_renderer.py +++ b/datalad/cli/tests/test_renderer.py @@ -21,6 +21,3 @@ def test_nadict(): d = nadict({1: 2}) eq_(d[1], 2) eq_(str(d[2]), NA_STRING) - - - diff --git a/datalad/cli/tests/test_utils.py b/datalad/cli/tests/test_utils.py index 044647458f..fcbfec5e59 100644 --- a/datalad/cli/tests/test_utils.py +++ b/datalad/cli/tests/test_utils.py @@ -40,6 +40,3 @@ def our_post_mortem(tb): our_exceptionhook(type_, value_, tb_) eq_(old_exceptionhook, sys.excepthook) - - - diff --git a/datalad/cli/utils.py b/datalad/cli/utils.py index 360dfcfeaa..848a5beeec 100644 --- a/datalad/cli/utils.py +++ b/datalad/cli/utils.py @@ -4,7 +4,6 @@ from datalad.log import is_interactive - _sys_excepthook = sys.excepthook # Just in case we ever need original one diff --git a/datalad/cmdline/__init__.py b/datalad/cmdline/__init__.py index 2e49b836bc..7a9ecb0eac 100644 --- a/datalad/cmdline/__init__.py +++ b/datalad/cmdline/__init__.py @@ -10,6 +10,7 @@ """ import warnings + warnings.warn( "All of datalad.cmdline is deprecated/discontinued as of datalad 0.16. " "A new CLI implementation is available at datalad.cli. " diff --git a/datalad/cmdline/helpers.py b/datalad/cmdline/helpers.py index 180e293db4..8bce47ac16 100644 --- a/datalad/cmdline/helpers.py +++ b/datalad/cmdline/helpers.py @@ -13,8 +13,8 @@ import os import warnings - from logging import getLogger + lgr = getLogger('datalad.cmdline') @@ -44,10 +44,10 @@ def get_repo_instance(path=os.curdir, class_=None): "It will be removed in a future release.", DeprecationWarning) - from datalad.utils import get_dataset_root from datalad.distribution.dataset import Dataset from datalad.support.annexrepo import AnnexRepo from datalad.support.gitrepo import GitRepo + from datalad.utils import get_dataset_root if class_ is not None: if class_ == AnnexRepo: diff --git a/datalad/cmdline/main.py b/datalad/cmdline/main.py index 0389e5b71e..f9cf79317a 100644 --- a/datalad/cmdline/main.py +++ b/datalad/cmdline/main.py @@ -12,6 +12,7 @@ """ import warnings + warnings.warn("datalad.cmdline.main was replaced by datalad.cli.main in " "datalad 0.16. Please update and reinstall extensions.", DeprecationWarning) diff --git a/datalad/config.py b/datalad/config.py index 0e9ed23432..eeec86f464 100644 --- a/datalad/config.py +++ b/datalad/config.py @@ -16,13 +16,13 @@ import threading import warnings from collections import namedtuple -from fasteners import InterProcessLock from functools import ( lru_cache, wraps, ) from pathlib import Path +from fasteners import InterProcessLock import datalad from datalad.consts import DATASET_CONFIG_FILE diff --git a/datalad/consts.py b/datalad/consts.py index c662e0c7d7..b245f3374a 100644 --- a/datalad/consts.py +++ b/datalad/consts.py @@ -10,8 +10,8 @@ """ import os -from os.path import join import re +from os.path import join # directory containing prepared metadata of a dataset repository: DATALAD_DOTDIR = ".datalad" diff --git a/datalad/core/distributed/clone.py b/datalad/core/distributed/clone.py index 52e1ad2d72..7576299f68 100644 --- a/datalad/core/distributed/clone.py +++ b/datalad/core/distributed/clone.py @@ -25,7 +25,7 @@ from datalad.interface.base import ( Interface, build_doc, - eval_results + eval_results, ) from datalad.interface.common_opts import ( location_description, @@ -45,12 +45,12 @@ ) from datalad.support.param import Parameter from datalad.utils import ( - knows_annex, PurePath, + knows_annex, rmtree, ) -from .clone_utils import ( # needed because other code imports it from here +from .clone_utils import ( # needed because other code imports it from here; RIA imports needed b/c datalad-next imports it from here ATM;; Remove after core was released and next dropped the ria patch. _check_autoenable_special_remotes, _format_clone_errors, _generate_candidate_clone_sources, @@ -60,8 +60,6 @@ _test_existing_clone_target, _try_clone_candidates, decode_source_spec, - # RIA imports needed b/c datalad-next imports it from here ATM; - # Remove after core was released and next dropped the ria patch. postclone_preannex_cfg_ria, postclonecfg_ria, ) diff --git a/datalad/core/distributed/clone_ria.py b/datalad/core/distributed/clone_ria.py index c78cc99b9a..8b1a4ff605 100644 --- a/datalad/core/distributed/clone_ria.py +++ b/datalad/core/distributed/clone_ria.py @@ -6,14 +6,14 @@ from typing import Dict from datalad.distribution.dataset import Dataset + +from . import clone as mod_clone # For now kept in clone_utils, to avoid circular import (see datalad-next) from .clone_utils import ( postclone_preannex_cfg_ria, postclonecfg_ria, ) -from . import clone as mod_clone - # we need to preserve the original functions to be able to call them # in the patch orig_post_git_init_processing_ = mod_clone._post_git_init_processing_ diff --git a/datalad/core/distributed/clone_utils.py b/datalad/core/distributed/clone_utils.py index b0682822a8..645256284a 100644 --- a/datalad/core/distributed/clone_utils.py +++ b/datalad/core/distributed/clone_utils.py @@ -32,7 +32,7 @@ from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import ( CapturedException, - DownloadError + DownloadError, ) from datalad.support.gitrepo import GitRepo from datalad.support.network import ( @@ -54,7 +54,6 @@ rmtree, ) - __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.core.distributed.clone') diff --git a/datalad/core/distributed/push.py b/datalad/core/distributed/push.py index e352987cb1..a7d4d8cd68 100644 --- a/datalad/core/distributed/push.py +++ b/datalad/core/distributed/push.py @@ -12,10 +12,18 @@ __docformat__ = 'restructuredtext' -from itertools import chain import logging import re +from itertools import chain +from datalad.core.local.diff import diff_dataset +from datalad.distribution.dataset import ( + Dataset, + EnsureDataset, + datasetmethod, + require_dataset, + resolve_path, +) from datalad.interface.base import ( Interface, build_doc, @@ -23,39 +31,27 @@ ) from datalad.interface.common_opts import ( jobs_opt, - recursion_limit, recursion_flag, + recursion_limit, ) -from datalad.interface.utils import render_action_summary from datalad.interface.results import annexjson2result +from datalad.interface.utils import render_action_summary from datalad.log import log_progress -from datalad.support.annexrepo import ( - AnnexRepo, -) -from datalad.support.gitrepo import GitRepo -from datalad.support.param import Parameter +from datalad.support.annexrepo import AnnexRepo from datalad.support.constraints import ( - EnsureStr, - EnsureNone, EnsureChoice, + EnsureNone, + EnsureStr, ) from datalad.support.exceptions import CommandError +from datalad.support.gitrepo import GitRepo +from datalad.support.param import Parameter from datalad.utils import ( Path, ensure_list, todo_interface_for_extensions, ) -from datalad.distribution.dataset import ( - Dataset, - EnsureDataset, - datasetmethod, - require_dataset, - resolve_path, -) -from datalad.core.local.diff import diff_dataset - - lgr = logging.getLogger('datalad.core.distributed.push') @@ -122,7 +118,7 @@ class Push(Interface): `git annex copy` altogether. 'auto' would use 'git annex copy' with '--auto' thus transferring only data which would satisfy "wanted" or "numcopies" settings for the remote (thus "nothing" otherwise). - 'auto-if-wanted' would enable '--auto' mode only if there is a + 'auto-if-wanted' would enable '--auto' mode only if there is a "wanted" setting for the remote, and transfer 'anything' otherwise. """, constraints=EnsureChoice( @@ -306,8 +302,8 @@ def custom_result_summary_renderer(results, action_summary): # pragma: more cov hints = set([r.get('hints', None) for r in results]) hints = [hint for hint in hints if hint is not None] if hints: - from datalad.ui import ui from datalad.support import ansi_colors + from datalad.ui import ui intro = ansi_colors.color_word( "Hints: ", ansi_colors.YELLOW) diff --git a/datalad/core/local/create.py b/datalad/core/local/create.py index 76885b57bd..82df22c5be 100644 --- a/datalad/core/local/create.py +++ b/datalad/core/local/create.py @@ -10,54 +10,48 @@ """ -import os import logging +import os +import os.path as op import random import uuid import warnings -from argparse import ( - REMAINDER, -) - +from argparse import REMAINDER from os import listdir -import os.path as op -from datalad import cfg -from datalad import _seed -from datalad.interface.base import Interface +from datalad import ( + _seed, + cfg, +) +from datalad.distribution.dataset import ( + Dataset, + EnsureDataset, + datasetmethod, + path_under_rev_dataset, + require_dataset, + resolve_path, +) from datalad.interface.base import ( + Interface, build_doc, eval_results, ) -from datalad.interface.common_opts import ( - location_description, -) +from datalad.interface.common_opts import location_description +from datalad.support.annexrepo import AnnexRepo from datalad.support.constraints import ( - EnsureStr, - EnsureNone, EnsureKeyChoice, + EnsureNone, + EnsureStr, ) +from datalad.support.gitrepo import GitRepo from datalad.support.param import Parameter from datalad.utils import ( - getpwd, + Path, ensure_list, get_dataset_root, - Path, -) - -from datalad.distribution.dataset import ( - Dataset, - datasetmethod, - EnsureDataset, - resolve_path, - path_under_rev_dataset, - require_dataset, + getpwd, ) -from datalad.support.gitrepo import GitRepo -from datalad.support.annexrepo import AnnexRepo - - __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.core.local.create') diff --git a/datalad/core/local/diff.py b/datalad/core/local/diff.py index 2a2a5295b3..ea3ed99509 100644 --- a/datalad/core/local/diff.py +++ b/datalad/core/local/diff.py @@ -13,37 +13,33 @@ import logging import os.path as op -from datalad.utils import ( - ensure_list, - ensure_unicode, - get_dataset_root, -) -from datalad.interface.base import ( - Interface, - build_doc, - eval_results, -) +from datalad.core.local.status import ( + Status, + _common_diffstatus_params, +) from datalad.distribution.dataset import ( Dataset, datasetmethod, + path_under_rev_dataset, require_dataset, resolve_path, - path_under_rev_dataset, ) - +from datalad.interface.base import ( + Interface, + build_doc, + eval_results, +) from datalad.support.constraints import ( EnsureNone, EnsureStr, ) +from datalad.support.exceptions import InvalidGitReferenceError from datalad.support.param import Parameter - -from datalad.core.local.status import ( - Status, - _common_diffstatus_params, -) -from datalad.support.exceptions import ( - InvalidGitReferenceError, +from datalad.utils import ( + ensure_list, + ensure_unicode, + get_dataset_root, ) lgr = logging.getLogger('datalad.core.local.diff') diff --git a/datalad/core/local/repo.py b/datalad/core/local/repo.py index 5ad1f57be7..c61f32e862 100644 --- a/datalad/core/local/repo.py +++ b/datalad/core/local/repo.py @@ -10,13 +10,14 @@ """ +import logging + from datalad.support.exceptions import ( - InvalidGitRepositoryError, InvalidAnnexRepositoryError, + InvalidGitRepositoryError, NoSuchPathError, ) -import logging lgr = logging.getLogger('datalad.core.local.repo') __all__ = ["repo_from_path"] @@ -43,8 +44,8 @@ def repo_from_path(path): """ # keep the imports local for now until it is clearer what the module setup # will be - from datalad.support.gitrepo import GitRepo from datalad.support.annexrepo import AnnexRepo + from datalad.support.gitrepo import GitRepo repo = None for cls, ckw, kw in ( diff --git a/datalad/core/local/resulthooks.py b/datalad/core/local/resulthooks.py index 5e1aaeb8a0..d49ad1233a 100644 --- a/datalad/core/local/resulthooks.py +++ b/datalad/core/local/resulthooks.py @@ -12,8 +12,8 @@ __docformat__ = 'restructuredtext' -import logging import json +import logging from datalad.support.exceptions import CapturedException diff --git a/datalad/core/local/status.py b/datalad/core/local/status.py index c591880aaa..1593b9dad8 100644 --- a/datalad/core/local/status.py +++ b/datalad/core/local/status.py @@ -14,45 +14,43 @@ import logging import os import os.path as op -from collections import OrderedDict import warnings +from collections import OrderedDict -from datalad.utils import ( - bytes2human, - ensure_list, - ensure_unicode, - get_dataset_root, +import datalad.support.ansi_colors as ac +import datalad.utils as ut +from datalad.distribution.dataset import ( + Dataset, + EnsureDataset, + datasetmethod, + path_under_rev_dataset, + require_dataset, + resolve_path, ) +from datalad.dochelpers import single_or_plural from datalad.interface.base import ( Interface, build_doc, eval_results, ) from datalad.interface.common_opts import ( - recursion_limit, recursion_flag, + recursion_limit, ) from datalad.interface.utils import generic_result_renderer -import datalad.support.ansi_colors as ac -from datalad.support.param import Parameter from datalad.support.constraints import ( EnsureChoice, EnsureNone, EnsureStr, ) -from datalad.distribution.dataset import ( - Dataset, - EnsureDataset, - datasetmethod, - require_dataset, - resolve_path, - path_under_rev_dataset, +from datalad.support.param import Parameter +from datalad.utils import ( + bytes2human, + ensure_list, + ensure_unicode, + get_dataset_root, ) -import datalad.utils as ut - -from datalad.dochelpers import single_or_plural - lgr = logging.getLogger('datalad.core.local.status') _common_diffstatus_params = dict( @@ -436,6 +434,7 @@ def custom_result_renderer(res, **kwargs): # pragma: more cover generic_result_renderer(res) return from datalad.ui import ui + # when to render relative paths: # 1) if a dataset arg was given # 2) if CWD is the refds diff --git a/datalad/coreapi.py b/datalad/coreapi.py index e575c15388..701659f979 100644 --- a/datalad/coreapi.py +++ b/datalad/coreapi.py @@ -26,8 +26,10 @@ def _generate_func_api(): from datalad.support.entrypoints import load_extensions load_extensions() - from .interface.base import get_interface_groups - from .interface.base import get_api_name + from .interface.base import ( + get_api_name, + get_interface_groups, + ) for grp_name, grp_descr, interfaces in get_interface_groups(): for intfspec in interfaces: diff --git a/datalad/customremotes/__init__.py b/datalad/customremotes/__init__.py index f232d5b870..dadc68e9c4 100644 --- a/datalad/customremotes/__init__.py +++ b/datalad/customremotes/__init__.py @@ -14,11 +14,10 @@ __all__ = ['RemoteError', 'SpecialRemote'] -from annexremote import ( - ProtocolError, - SpecialRemote as _SpecialRemote, - RemoteError as _RemoteError, -) +from annexremote import ProtocolError +from annexremote import RemoteError as _RemoteError +from annexremote import SpecialRemote as _SpecialRemote + from datalad.support.exceptions import format_exception_with_cause from datalad.ui import ui diff --git a/datalad/customremotes/base.py b/datalad/customremotes/base.py index e90d90e8de..40e349bd68 100644 --- a/datalad/customremotes/base.py +++ b/datalad/customremotes/base.py @@ -22,6 +22,7 @@ RemoteError, UnsupportedRequest, ) + from datalad.customremotes import SpecialRemote URI_PREFIX = "dl" diff --git a/datalad/customremotes/main.py b/datalad/customremotes/main.py index 7389728543..c88d9abaea 100644 --- a/datalad/customremotes/main.py +++ b/datalad/customremotes/main.py @@ -12,6 +12,7 @@ import argparse +import logging from datalad.cli.parser import ( parser_add_common_opt, @@ -20,7 +21,6 @@ from datalad.cli.utils import setup_exceptionhook from datalad.ui import ui -import logging lgr = logging.getLogger('datalad.customremotes') @@ -60,7 +60,9 @@ def _main(args, cls): def main(args=None, cls=None, remote_name=None, description=None): import sys + from datalad.support.entrypoints import load_extensions + # load extensions requested by configuration # analog to what coreapi is doing for a Python session # importantly, load them prior to parser construction, such diff --git a/datalad/customremotes/ria_utils.py b/datalad/customremotes/ria_utils.py index d009ea4737..a5b14d5338 100644 --- a/datalad/customremotes/ria_utils.py +++ b/datalad/customremotes/ria_utils.py @@ -12,7 +12,6 @@ import logging from pathlib import Path - lgr = logging.getLogger('datalad.customremotes.ria_utils') diff --git a/datalad/customremotes/tests/test_main.py b/datalad/customremotes/tests/test_main.py index e852db74dd..3e17be2e33 100644 --- a/datalad/customremotes/tests/test_main.py +++ b/datalad/customremotes/tests/test_main.py @@ -10,11 +10,8 @@ import logging import os -import pytest - -from ..main import main -from ..base import AnnexCustomRemote +import pytest from datalad.tests.utils_pytest import ( patch, @@ -23,6 +20,9 @@ ) from datalad.ui import ui +from ..base import AnnexCustomRemote +from ..main import main + def test_erroring_out(): class TooAbstract(AnnexCustomRemote): diff --git a/datalad/dataset/gitrepo.py b/datalad/dataset/gitrepo.py index e85695461b..10c6fa4d66 100644 --- a/datalad/dataset/gitrepo.py +++ b/datalad/dataset/gitrepo.py @@ -29,8 +29,8 @@ from os.path import lexists from typing import Optional from weakref import ( + WeakValueDictionary, finalize, - WeakValueDictionary ) from datalad.cmd import ( @@ -59,12 +59,11 @@ PathKnownToRepositoryError, ) from datalad.utils import ( + Path, ensure_list, lock_if_required, - Path, ) - lgr = logging.getLogger('datalad.dataset.gitrepo') preferred_encoding = getpreferredencoding(do_setlocale=False) diff --git a/datalad/dataset/repo.py b/datalad/dataset/repo.py index 1fb8bdc82a..0619d0004c 100644 --- a/datalad/dataset/repo.py +++ b/datalad/dataset/repo.py @@ -13,9 +13,9 @@ import logging import threading +from datalad import utils as ut from datalad.support.exceptions import InvalidInstanceRequestError from datalad.support.network import RI -from datalad import utils as ut lgr = logging.getLogger('datalad.repo') @@ -301,4 +301,3 @@ def __repr__(self): cls._repr = None cls.__repr__ = __repr__ return cls - diff --git a/datalad/distributed/create_sibling_ghlike.py b/datalad/distributed/create_sibling_ghlike.py index aa3ebb5af9..9cd5f42361 100644 --- a/datalad/distributed/create_sibling_ghlike.py +++ b/datalad/distributed/create_sibling_ghlike.py @@ -40,7 +40,6 @@ from datalad.ui import ui from datalad.utils import todo_interface_for_extensions - lgr = logging.getLogger('datalad.distributed.create_sibling_ghlike') diff --git a/datalad/distributed/create_sibling_gin.py b/datalad/distributed/create_sibling_gin.py index 6cf73b49d2..2c58d4ca52 100644 --- a/datalad/distributed/create_sibling_gin.py +++ b/datalad/distributed/create_sibling_gin.py @@ -14,8 +14,8 @@ from datalad.distributed.create_sibling_ghlike import _create_sibling from datalad.distributed.create_sibling_gogs import _GOGS from datalad.distribution.dataset import ( - datasetmethod, Dataset, + datasetmethod, ) from datalad.interface.base import ( Interface, @@ -24,7 +24,6 @@ ) from datalad.support.annexrepo import AnnexRepo - lgr = logging.getLogger('datalad.distributed.create_sibling_gin') diff --git a/datalad/distributed/drop.py b/datalad/distributed/drop.py index 8bc7047e37..35388280db 100644 --- a/datalad/distributed/drop.py +++ b/datalad/distributed/drop.py @@ -13,8 +13,8 @@ __docformat__ = 'restructuredtext' import logging -from itertools import chain import warnings +from itertools import chain from datalad.core.local.status import get_paths_by_ds from datalad.distribution.dataset import ( diff --git a/datalad/distributed/export_archive_ora.py b/datalad/distributed/export_archive_ora.py index b4aefebcf8..8ae90332cc 100644 --- a/datalad/distributed/export_archive_ora.py +++ b/datalad/distributed/export_archive_ora.py @@ -18,32 +18,30 @@ import subprocess from argparse import REMAINDER -from datalad.utils import ( - ensure_list, - rmtree, +from datalad.distribution.dataset import ( + EnsureDataset, + datasetmethod, + require_dataset, + resolve_path, ) from datalad.interface.base import ( Interface, build_doc, eval_results, ) -from datalad.interface.results import ( - get_status_dict, -) -from datalad.support.param import Parameter +from datalad.interface.results import get_status_dict +from datalad.log import log_progress from datalad.support.constraints import ( EnsureChoice, EnsureNone, EnsureStr, ) from datalad.support.exceptions import CapturedException -from datalad.distribution.dataset import ( - EnsureDataset, - datasetmethod, - require_dataset, - resolve_path, +from datalad.support.param import Parameter +from datalad.utils import ( + ensure_list, + rmtree, ) -from datalad.log import log_progress lgr = logging.getLogger('datalad.customremotes.export_archive_ora') diff --git a/datalad/distributed/export_to_figshare.py b/datalad/distributed/export_to_figshare.py index 81e24edd8e..81f06065a5 100644 --- a/datalad/distributed/export_to_figshare.py +++ b/datalad/distributed/export_to_figshare.py @@ -10,12 +10,15 @@ __docformat__ = 'restructuredtext' -from datalad.utils import unlink -from datalad.interface.base import Interface -from datalad.interface.base import build_doc +import logging + +from datalad.interface.base import ( + Interface, + build_doc, +) from datalad.interface.results import get_status_dict +from datalad.utils import unlink -import logging lgr = logging.getLogger('datalad.distributed.export_to_figshare') @@ -94,8 +97,9 @@ def upload_file(self, fname, files_url): # TODO: check if the file with the same name already available, and offer # to remove/prune it import os - from datalad.utils import md5sum + from datalad.ui import ui + from datalad.utils import md5sum file_rec = {'md5': md5sum(fname), 'name': os.path.basename(fname), 'size': os.stat(fname).st_size @@ -188,16 +192,18 @@ class ExportToFigshare(Interface): navigator. """ - from datalad.support.param import Parameter - from datalad.distribution.dataset import datasetmethod + from datalad.distribution.dataset import ( + EnsureDataset, + datasetmethod, + ) from datalad.interface.base import eval_results - from datalad.distribution.dataset import EnsureDataset from datalad.support.constraints import ( EnsureChoice, EnsureInt, EnsureNone, EnsureStr, ) + from datalad.support.param import Parameter _params_ = dict( dataset=Parameter( @@ -259,11 +265,13 @@ def __call__(filename=None, import logging lgr = logging.getLogger('datalad.plugin.export_to_figshare') - from datalad.ui import ui - from datalad.api import add_archive_content - from datalad.api import export_archive + from datalad.api import ( + add_archive_content, + export_archive, + ) from datalad.distribution.dataset import require_dataset from datalad.support.annexrepo import AnnexRepo + from datalad.ui import ui dataset = require_dataset(dataset, check_installed=True, purpose='export to figshare') diff --git a/datalad/distributed/ora_remote.py b/datalad/distributed/ora_remote.py index 919714967f..e30bc3f9f7 100644 --- a/datalad/distributed/ora_remote.py +++ b/datalad/distributed/ora_remote.py @@ -1,18 +1,19 @@ import functools +import logging import os +import shutil import stat +import subprocess import sys +from contextlib import contextmanager +from functools import wraps from pathlib import ( Path, - PurePosixPath + PurePosixPath, ) -from contextlib import contextmanager -import requests -import shutil from shlex import quote as sh_quote -import subprocess -import logging -from functools import wraps + +import requests from datalad import ssh_manager from datalad.config import anything2bool @@ -22,26 +23,25 @@ SpecialRemote, ) from datalad.customremotes.main import main as super_main +from datalad.customremotes.ria_utils import ( + UnknownLayoutVersion, + get_layout_locations, + verify_ria_url, +) from datalad.support.annex_utils import _sanitize_key from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import ( AccessDeniedError, AccessFailedError, CapturedException, - DownloadError + DownloadError, ) from datalad.support.network import url_path2local_path -from datalad.customremotes.ria_utils import ( - get_layout_locations, - UnknownLayoutVersion, - verify_ria_url, -) from datalad.utils import ( ensure_write_permission, - on_osx + on_osx, ) - lgr = logging.getLogger('datalad.customremotes.ria_remote') DEFAULT_BUFFER_SIZE = 65536 @@ -285,7 +285,12 @@ def write_file(self, file_path, content, mode='w'): f.write(content) def get_7z(self): - from datalad.cmd import CommandError, StdOutErrCapture, WitlessRunner + from datalad.cmd import ( + CommandError, + StdOutErrCapture, + WitlessRunner, + ) + # from datalad.utils import on_windows runner = WitlessRunner() @@ -1441,7 +1446,10 @@ def push_io(self): self.buffer_size) # We have a new instance. Kill the existing one and replace. - from atexit import register, unregister + from atexit import ( + register, + unregister, + ) if hasattr(self.io, 'close'): unregister(self.io.close) self.io.close() diff --git a/datalad/distributed/tests/test_create_sibling_gitlab.py b/datalad/distributed/tests/test_create_sibling_gitlab.py index a2c9a5d130..feee5624ec 100644 --- a/datalad/distributed/tests/test_create_sibling_gitlab.py +++ b/datalad/distributed/tests/test_create_sibling_gitlab.py @@ -8,7 +8,9 @@ """Test create publication target on gitlab""" import os + import pytest + # this must import ok with and without gitlab from datalad.api import ( Dataset, diff --git a/datalad/distributed/tests/test_drop.py b/datalad/distributed/tests/test_drop.py index a74d45ab09..78accaf8ab 100644 --- a/datalad/distributed/tests/test_drop.py +++ b/datalad/distributed/tests/test_drop.py @@ -46,7 +46,6 @@ ) from datalad.utils import chpwd - ckwa = dict(result_renderer='disabled') diff --git a/datalad/distribution/create_sibling.py b/datalad/distribution/create_sibling.py index 7ae1d966c2..f31028d8e8 100644 --- a/datalad/distribution/create_sibling.py +++ b/datalad/distribution/create_sibling.py @@ -13,8 +13,8 @@ import logging import os -from os.path import curdir import shlex +from os.path import curdir from os.path import join as opj from os.path import ( normpath, diff --git a/datalad/distribution/create_sibling_github.py b/datalad/distribution/create_sibling_github.py index 744b32eff6..d1c7ef864c 100644 --- a/datalad/distribution/create_sibling_github.py +++ b/datalad/distribution/create_sibling_github.py @@ -9,6 +9,7 @@ """Import shim to ease the transition after the move under distributed/""" import warnings + warnings.warn( "CreateSiblingGithub has been moved to " "datalad.distributed.create_sibling_github. Please adjust the import.", diff --git a/datalad/distribution/create_test_dataset.py b/datalad/distribution/create_test_dataset.py index af653aecec..382499aa52 100644 --- a/datalad/distribution/create_test_dataset.py +++ b/datalad/distribution/create_test_dataset.py @@ -13,20 +13,31 @@ __docformat__ = 'numpy' -import random import logging +import os +import random import tempfile +from os.path import ( + abspath, + exists, + isabs, +) +from os.path import join as opj -from datalad.utils import get_tempfile_kwargs -import os -from os.path import join as opj, exists, isabs, abspath from datalad.distribution.dataset import Dataset -from datalad.support.param import Parameter -from datalad.support.constraints import EnsureStr, EnsureNone, EnsureInt -from datalad.support.gitrepo import GitRepo +from datalad.interface.base import ( + Interface, + build_doc, +) from datalad.support.annexrepo import AnnexRepo -from datalad.interface.base import Interface -from datalad.interface.base import build_doc +from datalad.support.constraints import ( + EnsureInt, + EnsureNone, + EnsureStr, +) +from datalad.support.gitrepo import GitRepo +from datalad.support.param import Parameter +from datalad.utils import get_tempfile_kwargs lgr = logging.getLogger('datalad.distribution.tests') @@ -85,6 +96,7 @@ def _makeds(path, levels, ds=None, max_leading_dirs=2): """ # we apparently can't import api functionality within api from datalad.api import save + # To simplify managing all the file paths etc if not isabs(path): path = abspath(path) diff --git a/datalad/distribution/dataset.py b/datalad/distribution/dataset.py index 029cf507b8..9186d1df48 100644 --- a/datalad/distribution/dataset.py +++ b/datalad/distribution/dataset.py @@ -15,47 +15,41 @@ from os.path import ( curdir, exists, - join as opj, +) +from os.path import join as opj +from os.path import ( normpath, pardir, ) from weakref import WeakValueDictionary +import datalad.utils as ut from datalad import cfg from datalad.config import ConfigManager from datalad.core.local.repo import repo_from_path +from datalad.dataset.repo import ( + PathBasedFlyweight, + path_based_str_repr, +) +from datalad.support import path as op from datalad.support.annexrepo import AnnexRepo from datalad.support.constraints import Constraint # DueCredit from datalad.support.due import due from datalad.support.due_utils import duecredit_dataset -from datalad.support.exceptions import ( - NoDatasetFound, -) -from datalad.dataset.repo import ( - path_based_str_repr, - PathBasedFlyweight, -) -from datalad.support.gitrepo import ( - GitRepo, -) -from datalad.support import path as op - -import datalad.utils as ut +from datalad.support.exceptions import NoDatasetFound +from datalad.support.gitrepo import GitRepo +from datalad.utils import \ + get_dataset_root # TODO remove after a while, when external consumers have adjusted; to use get_dataset_root() from datalad.utils import ( - getpwd, - optional_args, - get_dataset_root, - get_sig_param_names, - # TODO remove after a while, when external consumers have adjusted - # to use get_dataset_root() - get_dataset_root as rev_get_dataset_root, Path, PurePath, ensure_list, + get_sig_param_names, + getpwd, + optional_args, ) - lgr = logging.getLogger('datalad.dataset') lgr.log(5, "Importing dataset") @@ -188,6 +182,7 @@ def __getattr__(self, attr): # load entire datalad.api which will also bind datasetmethods # from extensions. import datalad.api + # which would bind all known interfaces as well. # Although adds overhead, good for UX return super(Dataset, self).__getattribute__(attr) diff --git a/datalad/distribution/get.py b/datalad/distribution/get.py index a5a927548a..ea4277332f 100644 --- a/datalad/distribution/get.py +++ b/datalad/distribution/get.py @@ -11,74 +11,69 @@ """ import logging -import re - import os.path as op +import re from datalad.config import ConfigManager +from datalad.core.distributed.clone import clone_dataset +from datalad.distribution.dataset import ( + Dataset, + EnsureDataset, + datasetmethod, + require_dataset, +) +from datalad.distribution.utils import _get_flexible_source_candidates from datalad.interface.base import ( Interface, + build_doc, eval_results, ) -from datalad.interface.base import build_doc +from datalad.interface.common_opts import ( + jobs_opt, + location_description, + reckless_opt, + recursion_flag, +) from datalad.interface.results import ( + annexjson2result, get_status_dict, + is_ok_dataset, + results_from_annex_noinfo, results_from_paths, - annexjson2result, success_status_map, - results_from_annex_noinfo, -) -from datalad.interface.common_opts import ( - recursion_flag, - location_description, - jobs_opt, - reckless_opt, ) -from datalad.interface.results import is_ok_dataset +from datalad.local.subdatasets import Subdatasets +from datalad.support.annexrepo import AnnexRepo +from datalad.support.collections import ReadOnlyDict from datalad.support.constraints import ( - EnsureInt, EnsureChoice, - EnsureStr, + EnsureInt, EnsureNone, -) -from datalad.support.collections import ReadOnlyDict -from datalad.support.param import Parameter -from datalad.support.annexrepo import AnnexRepo -from datalad.support.gitrepo import ( - GitRepo, - _fixup_submodule_dotgit_setup, + EnsureStr, ) from datalad.support.exceptions import ( CapturedException, CommandError, InsufficientArgumentsError, ) +from datalad.support.gitrepo import ( + GitRepo, + _fixup_submodule_dotgit_setup, +) from datalad.support.network import ( - URL, RI, + URL, urlquote, ) -from datalad.support.parallel import ( - ProducerConsumerProgressLog, -) +from datalad.support.parallel import ProducerConsumerProgressLog +from datalad.support.param import Parameter from datalad.utils import ( - unique, Path, get_dataset_root, shortened_repr, + unique, ) -from datalad.local.subdatasets import Subdatasets - -from datalad.distribution.dataset import ( - Dataset, - EnsureDataset, - datasetmethod, - require_dataset, -) -from datalad.core.distributed.clone import clone_dataset -from datalad.distribution.utils import _get_flexible_source_candidates - __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.distribution.get') @@ -881,7 +876,7 @@ def __call__( reckless=None, jobs='auto', ): - + if not (dataset or path): raise InsufficientArgumentsError( "Neither dataset nor target path(s) provided") diff --git a/datalad/distribution/install.py b/datalad/distribution/install.py index 1d7e08fa20..f93454d210 100644 --- a/datalad/distribution/install.py +++ b/datalad/distribution/install.py @@ -13,23 +13,31 @@ import logging from os import curdir -from datalad.interface.base import Interface +from datalad.core.distributed.clone import Clone +from datalad.distribution.dataset import ( + EnsureDataset, + datasetmethod, + require_dataset, + resolve_path, +) +from datalad.distribution.get import Get +from datalad.interface.base import ( + Interface, + build_doc, + eval_results, +) from datalad.interface.common_opts import ( - recursion_flag, - recursion_limit, - location_description, jobs_opt, + location_description, reckless_opt, + recursion_flag, + recursion_limit, ) from datalad.interface.results import ( - get_status_dict, YieldDatasets, + get_status_dict, is_result_matching_pathsource_argument, ) -from datalad.interface.base import ( - build_doc, - eval_results, -) from datalad.support.constraints import ( EnsureNone, EnsureStr, @@ -38,22 +46,13 @@ CapturedException, InsufficientArgumentsError, ) -from datalad.support.param import Parameter from datalad.support.network import ( RI, PathRI, ) +from datalad.support.param import Parameter from datalad.utils import ensure_list -from datalad.distribution.dataset import ( - datasetmethod, - resolve_path, - require_dataset, - EnsureDataset, -) -from datalad.distribution.get import Get -from datalad.core.distributed.clone import Clone - __docformat__ = 'restructuredtext' lgr = logging.getLogger('datalad.distribution.install') diff --git a/datalad/distribution/tests/test_create_sibling.py b/datalad/distribution/tests/test_create_sibling.py index 02cb13589b..dbcfb6b5d7 100644 --- a/datalad/distribution/tests/test_create_sibling.py +++ b/datalad/distribution/tests/test_create_sibling.py @@ -37,7 +37,6 @@ ) from datalad.support.gitrepo import GitRepo from datalad.support.network import urlquote -from datalad.tests.utils_testdatasets import _mk_submodule_annex from datalad.tests.utils_pytest import ( DEFAULT_BRANCH, DEFAULT_REMOTE, @@ -70,6 +69,7 @@ with_tempfile, with_testsui, ) +from datalad.tests.utils_testdatasets import _mk_submodule_annex from datalad.utils import ( Path, _path_, diff --git a/datalad/distribution/tests/test_dataset.py b/datalad/distribution/tests/test_dataset.py index 63ad84cd78..5f31954a21 100644 --- a/datalad/distribution/tests/test_dataset.py +++ b/datalad/distribution/tests/test_dataset.py @@ -28,9 +28,7 @@ get, ) from datalad.support.annexrepo import AnnexRepo -from datalad.support.exceptions import ( - NoDatasetFound, -) +from datalad.support.exceptions import NoDatasetFound from datalad.support.gitrepo import GitRepo from datalad.tests.utils_pytest import ( OBSCURE_FILENAME, diff --git a/datalad/distribution/tests/test_get.py b/datalad/distribution/tests/test_get.py index 4593c5cd42..c439714c7b 100644 --- a/datalad/distribution/tests/test_get.py +++ b/datalad/distribution/tests/test_get.py @@ -32,34 +32,34 @@ RemoteNotAvailableError, ) from datalad.support.network import get_local_file_url -from datalad.tests.utils_testdatasets import ( - _make_dataset_hierarchy, - _mk_submodule_annex, -) from datalad.tests.utils_pytest import ( - create_tree, + DEFAULT_REMOTE, assert_false, - assert_raises, assert_in, - assert_status, assert_in_results, + assert_message, assert_not_in_results, + assert_raises, assert_repo_status, assert_result_count, - assert_message, - DEFAULT_REMOTE, + assert_status, + create_tree, eq_, - known_failure_windows, known_failure_githubci_win, + known_failure_windows, ok_, serve_path_via_http, skip_if_adjusted_branch, - skip_ssh, skip_if_on_windows, + skip_ssh, slow, with_tempfile, with_tree, ) +from datalad.tests.utils_testdatasets import ( + _make_dataset_hierarchy, + _mk_submodule_annex, +) from datalad.utils import ( Path, chpwd, diff --git a/datalad/distribution/tests/test_install.py b/datalad/distribution/tests/test_install.py index 2741ff962e..6b2f8986e4 100644 --- a/datalad/distribution/tests/test_install.py +++ b/datalad/distribution/tests/test_install.py @@ -38,10 +38,6 @@ ) from datalad.support.gitrepo import GitRepo from datalad.support.network import get_local_file_url -from datalad.tests.utils_testdatasets import ( - _make_dataset_hierarchy, - _mk_submodule_annex, -) from datalad.tests.utils_pytest import ( DEFAULT_BRANCH, DEFAULT_REMOTE, @@ -75,6 +71,10 @@ with_tempfile, with_tree, ) +from datalad.tests.utils_testdatasets import ( + _make_dataset_hierarchy, + _mk_submodule_annex, +) from datalad.utils import ( Path, _path_, diff --git a/datalad/distribution/tests/test_siblings.py b/datalad/distribution/tests/test_siblings.py index fec54b84f9..47c36c1595 100644 --- a/datalad/distribution/tests/test_siblings.py +++ b/datalad/distribution/tests/test_siblings.py @@ -9,6 +9,8 @@ """ +import pytest + from datalad.api import ( Dataset, clone, @@ -48,8 +50,6 @@ on_windows, ) -import pytest - @with_tempfile(mkdir=True) @with_tempfile(mkdir=True) diff --git a/datalad/distribution/uninstall.py b/datalad/distribution/uninstall.py index 06914b5b85..8652cf9f86 100644 --- a/datalad/distribution/uninstall.py +++ b/datalad/distribution/uninstall.py @@ -14,33 +14,30 @@ import logging -from datalad.support.param import Parameter -from datalad.support.constraints import ( - EnsureStr, - EnsureNone, -) +from datalad.core.local.status import get_paths_by_ds from datalad.distribution.dataset import ( - datasetmethod, - require_dataset, Dataset, EnsureDataset, + datasetmethod, + require_dataset, +) +from datalad.interface.base import ( + Interface, + build_doc, + eval_results, ) -from datalad.interface.base import Interface from datalad.interface.common_opts import ( if_dirty_opt, recursion_flag, ) -from datalad.interface.utils import handle_dirty_dataset from datalad.interface.results import get_status_dict -from datalad.interface.base import ( - build_doc, - eval_results, -) -from datalad.utils import ( - ensure_list, +from datalad.interface.utils import handle_dirty_dataset +from datalad.support.constraints import ( + EnsureNone, + EnsureStr, ) -from datalad.core.local.status import get_paths_by_ds - +from datalad.support.param import Parameter +from datalad.utils import ensure_list lgr = logging.getLogger('datalad.distribution.uninstall') diff --git a/datalad/distribution/update.py b/datalad/distribution/update.py index 2fc2235e33..6cfea43315 100644 --- a/datalad/distribution/update.py +++ b/datalad/distribution/update.py @@ -13,36 +13,36 @@ __docformat__ = 'restructuredtext' +import itertools import logging from os.path import lexists -import itertools -from datalad.interface.base import Interface +from datalad.distribution.dataset import require_dataset from datalad.interface.base import ( + Interface, build_doc, eval_results, ) +from datalad.interface.common_opts import ( + recursion_flag, + recursion_limit, +) from datalad.interface.results import ( + YieldDatasets, get_status_dict, - YieldDatasets ) +from datalad.support.annexrepo import AnnexRepo from datalad.support.constraints import ( EnsureBool, EnsureChoice, - EnsureStr, EnsureNone, + EnsureStr, ) -from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import ( CapturedException, - CommandError + CommandError, ) from datalad.support.param import Parameter -from datalad.interface.common_opts import ( - recursion_flag, - recursion_limit, -) -from datalad.distribution.dataset import require_dataset from .dataset import ( EnsureDataset, diff --git a/datalad/distribution/utils.py b/datalad/distribution/utils.py index 3ef3ca4be0..701578640a 100644 --- a/datalad/distribution/utils.py +++ b/datalad/distribution/utils.py @@ -11,22 +11,19 @@ """ import logging - -from os.path import ( - isabs, - join as opj, - normpath, -) import posixpath +from os.path import isabs +from os.path import join as opj +from os.path import normpath + from datalad.log import log_progress from datalad.support.annexrepo import AnnexRepo from datalad.support.network import ( - PathRI, RI, URL, + PathRI, ) - lgr = logging.getLogger('datalad.distribution.utils') diff --git a/datalad/dochelpers.py b/datalad/dochelpers.py index cca9ef6324..cb30491d4f 100644 --- a/datalad/dochelpers.py +++ b/datalad/dochelpers.py @@ -13,10 +13,10 @@ """ import logging -import re -import textwrap import os +import re import sys +import textwrap import traceback from datalad.support.exceptions import CapturedException diff --git a/datalad/downloaders/__init__.py b/datalad/downloaders/__init__.py index a350edecbe..d35f50d278 100644 --- a/datalad/downloaders/__init__.py +++ b/datalad/downloaders/__init__.py @@ -12,15 +12,17 @@ from datalad.downloaders.credentials import ( AWS_S3, - LORIS_Token, NDA_S3, + GitCredential, + LORIS_Token, Token, UserPassword, - GitCredential) +) __docformat__ = 'restructuredtext' from logging import getLogger + lgr = getLogger('datalad.providers') # TODO: we might not need to instantiate it right here diff --git a/datalad/downloaders/base.py b/datalad/downloaders/base.py index 1fe2f38c3d..51e4deee62 100644 --- a/datalad/downloaders/base.py +++ b/datalad/downloaders/base.py @@ -12,27 +12,26 @@ __docformat__ = 'restructuredtext' -import msgpack import os +import os.path as op import sys import time +from abc import ( + ABCMeta, + abstractmethod, +) +from logging import getLogger +from os.path import ( + exists, + isdir, +) +from os.path import join as opj -from abc import ABCMeta, abstractmethod -import os.path as op -from os.path import exists, join as opj, isdir +import msgpack +from datalad.downloaders import CREDENTIAL_TYPES from .. import cfg -from ..ui import ui -from ..utils import ( - auto_repr, - ensure_unicode, - unlink, -) -from .credentials import ( - CompositeCredential, -) -from datalad.downloaders import CREDENTIAL_TYPES from ..support.exceptions import ( AccessDeniedError, AccessPermissionExpiredError, @@ -47,8 +46,14 @@ try_lock, try_lock_informatively, ) +from ..ui import ui +from ..utils import ( + auto_repr, + ensure_unicode, + unlink, +) +from .credentials import CompositeCredential -from logging import getLogger lgr = getLogger('datalad.downloaders') @@ -538,6 +543,7 @@ def cache(self): # TODO: move this all logic outside into a dedicated caching beast lgr.info("Initializing cache for fetches") import dbm + # Initiate cache. # Very rudimentary caching for now, might fail many ways cache_dir = cfg.obtain('datalad.locations.cache') diff --git a/datalad/downloaders/configs/indi.cfg b/datalad/downloaders/configs/indi.cfg index 109aa6da87..db7ab86952 100644 --- a/datalad/downloaders/configs/indi.cfg +++ b/datalad/downloaders/configs/indi.cfg @@ -3,5 +3,3 @@ [provider:indi-s3] url_re = s3://fcp-indi($|/.*) authentication_type = aws-s3 - - diff --git a/datalad/downloaders/configs/loris.cfg b/datalad/downloaders/configs/loris.cfg index 252310ca42..ac88a3e191 100644 --- a/datalad/downloaders/configs/loris.cfg +++ b/datalad/downloaders/configs/loris.cfg @@ -7,4 +7,3 @@ loris-token_failure_re = "User not authenticated"}$ [credential:loris-demo] url = https://demo.loris.ca/api/v0.0.2/login type = loris-token - diff --git a/datalad/downloaders/configs/openfmri.cfg b/datalad/downloaders/configs/openfmri.cfg index 73a091cc8c..79c4ec93b3 100644 --- a/datalad/downloaders/configs/openfmri.cfg +++ b/datalad/downloaders/configs/openfmri.cfg @@ -9,5 +9,3 @@ authentication_type = none url_re = s3://((|traverses3-)openfmri|openneuro)($|/.*) credential = datalad-test-s3 authentication_type = aws-s3 - - diff --git a/datalad/downloaders/credentials.py b/datalad/downloaders/credentials.py index bf21a62296..302cfd27b3 100644 --- a/datalad/downloaders/credentials.py +++ b/datalad/downloaders/credentials.py @@ -21,22 +21,22 @@ """ import time +from logging import getLogger + +from datalad import cfg as dlcfg +from datalad.config import anything2bool +from datalad.distribution.dataset import Dataset +from datalad.local.gitcredential import GitCredentialInterface from ..support.exceptions import ( AccessDeniedError, CapturedException, ) from ..support.keyring_ import keyring as keyring_ +from ..support.network import iso8601_to_epoch from ..ui import ui from ..utils import auto_repr -from ..support.network import iso8601_to_epoch -from datalad import cfg as dlcfg -from datalad.config import anything2bool -from datalad.distribution.dataset import Dataset -from datalad.local.gitcredential import GitCredentialInterface - -from logging import getLogger lgr = getLogger('datalad.downloaders.credentials') @@ -399,6 +399,7 @@ def __call__(self): def _nda_adapter(composite, user=None, password=None): from datalad.support.third.nda_aws_token_generator import NDATokenGenerator + from .. import cfg nda_auth_url = cfg.obtain('datalad.externals.nda.dbserver') gen = NDATokenGenerator(nda_auth_url) diff --git a/datalad/downloaders/http.py b/datalad/downloaders/http.py index 033629e67e..c116486770 100644 --- a/datalad/downloaders/http.py +++ b/datalad/downloaders/http.py @@ -9,33 +9,19 @@ """Provide access to stuff (html, data files) via HTTP and HTTPS """ +import io import re +from logging import getLogger +from time import sleep + import requests import requests.auth from requests.utils import parse_dict_header -# at some point was trying to be too specific about which exceptions to -# catch for a retry of a download. -# from urllib3.exceptions import MaxRetryError, NewConnectionError - -import io -from time import sleep - from .. import __version__ -from ..utils import ( - ensure_list_from_str, - ensure_dict_from_str, - ensure_bytes, -) from ..dochelpers import borrowkwargs - -from ..ui import ui -from ..utils import auto_repr -from ..support.network import get_url_filename -from ..support.network import get_response_disposition_filename -from ..support.network import rfc2822_to_epoch +from ..log import LoggerHelper from ..support.cookies import cookies_db -from ..support.status import FileStatus from ..support.exceptions import ( AccessDeniedError, AccessFailedError, @@ -43,12 +29,33 @@ DownloadError, UnhandledRedirectError, ) +from ..support.network import ( + get_response_disposition_filename, + get_url_filename, + rfc2822_to_epoch, +) +from ..support.status import FileStatus +from ..ui import ui +from ..utils import ( + auto_repr, + ensure_bytes, + ensure_dict_from_str, + ensure_list_from_str, +) +from .base import ( + Authenticator, + BaseDownloader, + DownloaderSession, +) + +# at some point was trying to be too specific about which exceptions to +# catch for a retry of a download. +# from urllib3.exceptions import MaxRetryError, NewConnectionError + + + -from .base import Authenticator -from .base import BaseDownloader, DownloaderSession -from logging import getLogger -from ..log import LoggerHelper lgr = getLogger('datalad.http') # Following https://meta.wikimedia.org/wiki/User-Agent_policy to provide @@ -74,6 +81,7 @@ # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA. # The only thing missing will be the response.body which is not logged. import http.client + # TODO: nohow wrapped with logging, plain prints (heh heh), so formatting will not be consistent http.client.HTTPConnection.debuglevel = 1 diff --git a/datalad/downloaders/providers.py b/datalad/downloaders/providers.py index cd0df20b42..ca909d8c10 100644 --- a/datalad/downloaders/providers.py +++ b/datalad/downloaders/providers.py @@ -9,39 +9,47 @@ """Data providers - bind downloaders and credentials together """ +import re +from configparser import ConfigParser as SafeConfigParserWithIncludes from glob import glob from logging import getLogger - -import re -from os.path import dirname, abspath, join as pathjoin +from os.path import ( + abspath, + dirname, +) +from os.path import join as pathjoin from urllib.parse import urlparse -from .base import NoneAuthenticator, NotImplementedAuthenticator +from datalad.support import path +from datalad.support.external_versions import external_versions +from datalad.support.network import RI +from datalad.utils import ( + Path, + auto_repr, + ensure_list_from_str, + get_dataset_root, +) +from ..interface.common_cfg import dirs +from .base import ( + NoneAuthenticator, + NotImplementedAuthenticator, +) from .http import ( HTMLFormAuthenticator, HTTPAnonBearerTokenAuthenticator, HTTPAuthAuthenticator, HTTPBasicAuthAuthenticator, - HTTPDigestAuthAuthenticator, HTTPBearerTokenAuthenticator, + HTTPDigestAuthAuthenticator, HTTPDownloader, HTTPTokenAuthenticator, ) -from .s3 import S3Authenticator, S3Downloader -from .shub import SHubDownloader -from configparser import ConfigParser as SafeConfigParserWithIncludes -from datalad.support.external_versions import external_versions -from datalad.support.network import RI -from datalad.support import path -from datalad.utils import ( - auto_repr, - ensure_list_from_str, - get_dataset_root, - Path, +from .s3 import ( + S3Authenticator, + S3Downloader, ) - -from ..interface.common_cfg import dirs +from .shub import SHubDownloader lgr = getLogger('datalad.downloaders.providers') diff --git a/datalad/downloaders/s3.py b/datalad/downloaders/s3.py index 88cb71fab5..ae9b1932d0 100644 --- a/datalad/downloaders/s3.py +++ b/datalad/downloaders/s3.py @@ -9,35 +9,34 @@ """Provide access to Amazon S3 objects. """ +import logging import re -from urllib.parse import urlsplit, unquote as urlunquote +from logging import getLogger from threading import Lock +from urllib.parse import unquote as urlunquote +from urllib.parse import urlsplit import boto3 import botocore -from ..utils import ( - auto_repr, - ensure_dict_from_str, -) -from ..dochelpers import ( - borrowkwargs, -) -from ..support.network import ( - get_url_straight_filename, -) - -from .base import Authenticator -from .base import BaseDownloader, DownloaderSession +from ..dochelpers import borrowkwargs from ..support.exceptions import ( AccessDeniedError, AccessPermissionExpiredError, TargetFileAbsent, ) +from ..support.network import get_url_straight_filename from ..support.status import FileStatus +from ..utils import ( + auto_repr, + ensure_dict_from_str, +) +from .base import ( + Authenticator, + BaseDownloader, + DownloaderSession, +) -import logging -from logging import getLogger lgr = getLogger('datalad.s3') boto_lgr = logging.getLogger('boto3') # not in effect at all, probably those are setup later @@ -45,7 +44,6 @@ import warnings - __docformat__ = 'restructuredtext' diff --git a/datalad/interface/add_archive_content.py b/datalad/interface/add_archive_content.py index d5861d6bf1..f482cab1ab 100644 --- a/datalad/interface/add_archive_content.py +++ b/datalad/interface/add_archive_content.py @@ -10,6 +10,7 @@ """ import warnings + warnings.warn( "AddArchiveContent has been moved to datalad.local.add_archive_content. " "This module was deprecated in 0.16.0, and will be removed in a future " diff --git a/datalad/interface/base.py b/datalad/interface/base.py index 019e138517..9caf6c0f8d 100644 --- a/datalad/interface/base.py +++ b/datalad/interface/base.py @@ -25,15 +25,14 @@ ABC, abstractmethod, ) - from functools import wraps from importlib import import_module from typing import ( + TYPE_CHECKING, Callable, Dict, Generator, TypeVar, - TYPE_CHECKING, ) import datalad @@ -50,11 +49,11 @@ from datalad.interface.common_opts import eval_params from datalad.interface.results import known_result_xfms from datalad.interface.utils import ( + _process_results, get_result_filter, keep_result, render_action_summary, xfm_result, - _process_results ) from datalad.support.exceptions import ( CapturedException, diff --git a/datalad/interface/clean.py b/datalad/interface/clean.py index 921c6f4f2a..6eaf4b7169 100644 --- a/datalad/interface/clean.py +++ b/datalad/interface/clean.py @@ -10,6 +10,7 @@ """ import warnings + warnings.warn( "Clean has been moved to datalad.local.clean. " "This module was deprecated in 0.16.0, and will be removed in a future " diff --git a/datalad/interface/common_cfg.py b/datalad/interface/common_cfg.py index 1a17045f07..fd04500dee 100644 --- a/datalad/interface/common_cfg.py +++ b/datalad/interface/common_cfg.py @@ -12,20 +12,20 @@ __docformat__ = 'restructuredtext' -from collections.abc import Mapping import logging +import time +from collections.abc import Mapping from os import environ from os.path import expanduser from os.path import join as opj -import time from platformdirs import AppDirs from datalad.support.constraints import ( EnsureBool, EnsureChoice, - EnsureInt, EnsureFloat, + EnsureInt, EnsureListOf, EnsureNone, EnsureStr, @@ -144,9 +144,10 @@ def update(self, *args, **kwargs): def get_default_ssh(): - from datalad.utils import on_windows from pathlib import Path + from datalad.utils import on_windows + if on_windows: windows_openssh_path = \ environ.get("WINDIR", r"C:\Windows") + r"\System32\OpenSSH\ssh.exe" diff --git a/datalad/interface/common_opts.py b/datalad/interface/common_opts.py index 18a81ad26b..66d4b50b40 100644 --- a/datalad/interface/common_opts.py +++ b/datalad/interface/common_opts.py @@ -13,7 +13,6 @@ __docformat__ = 'restructuredtext' from datalad.interface.results import known_result_xfms -from datalad.support.param import Parameter from datalad.support.constraints import ( EnsureBool, EnsureCallable, @@ -23,6 +22,7 @@ EnsureStr, EnsureStrPrefix, ) +from datalad.support.param import Parameter location_description = Parameter( args=("-D", "--description",), @@ -170,8 +170,8 @@ EnsureChoice(None, True, False, 'auto', 'ephemeral') | \ EnsureStrPrefix('shared-'), metavar='auto|ephemeral|shared-...', - doc="""Obtain a dataset or subdatset and set it up in a potentially - unsafe way for performance, or access reasons. + doc="""Obtain a dataset or subdatset and set it up in a potentially + unsafe way for performance, or access reasons. Use with care, any dataset is marked as 'untrusted'. The reckless mode is stored in a dataset's local configuration under 'datalad.clone.reckless', and will be inherited to any of its subdatasets. diff --git a/datalad/interface/download_url.py b/datalad/interface/download_url.py index f7051948f8..e69810c2a9 100644 --- a/datalad/interface/download_url.py +++ b/datalad/interface/download_url.py @@ -10,6 +10,7 @@ """ import warnings + warnings.warn( "DownloadURL has been moved to datalad.local.download_url. " "This module was deprecated in 0.16.0, and will be removed in a future " diff --git a/datalad/interface/rerun.py b/datalad/interface/rerun.py index 58cbec3064..435ab8d27f 100644 --- a/datalad/interface/rerun.py +++ b/datalad/interface/rerun.py @@ -10,6 +10,7 @@ """ import warnings + warnings.warn( "ReRun has been moved to datalad.local.rerun. " "This module was deprecated in 0.16.0, and will be removed in a future " diff --git a/datalad/interface/run_procedure.py b/datalad/interface/run_procedure.py index 50e1f0158d..cea73f89a6 100644 --- a/datalad/interface/run_procedure.py +++ b/datalad/interface/run_procedure.py @@ -10,6 +10,7 @@ """ import warnings + warnings.warn( "RunProcedure has been moved to datalad.local.run_procedure. " "This module was deprecated in 0.16.0, and will be removed in a future " diff --git a/datalad/interface/shell_completion.py b/datalad/interface/shell_completion.py index 85e3c23702..45f905cf93 100644 --- a/datalad/interface/shell_completion.py +++ b/datalad/interface/shell_completion.py @@ -12,7 +12,7 @@ __docformat__ = 'restructuredtext' -from .base import Interface +from logging import getLogger from datalad.interface.base import ( build_doc, @@ -21,7 +21,7 @@ from datalad.interface.results import get_status_dict from datalad.support.exceptions import CapturedException -from logging import getLogger +from .base import Interface lgr = getLogger("datalad.interface.shell_completion") diff --git a/datalad/interface/unlock.py b/datalad/interface/unlock.py index d99190ce72..624cc8ce31 100644 --- a/datalad/interface/unlock.py +++ b/datalad/interface/unlock.py @@ -10,6 +10,7 @@ """ import warnings + warnings.warn( "Unlock has been moved to datalad.local.unlock. " "This module was deprecated in 0.16.0, and will be removed in a future " diff --git a/datalad/interface/utils.py b/datalad/interface/utils.py index 9010f17b4a..2ddbabc553 100644 --- a/datalad/interface/utils.py +++ b/datalad/interface/utils.py @@ -12,36 +12,35 @@ __docformat__ = 'restructuredtext' +import json import logging import sys -from time import time from os import listdir -from os.path import join as opj from os.path import isdir -from os.path import relpath -from os.path import sep - -import json - -from typing import ( - TypeVar +from os.path import join as opj +from os.path import ( + relpath, + sep, ) +from time import time +from typing import TypeVar + +import datalad.support.ansi_colors as ac +from datalad import cfg as dlcfg +from datalad.dochelpers import single_or_plural +from datalad.support.exceptions import CapturedException +from datalad.support.gitrepo import GitRepo +from datalad.ui import ui # avoid import from API to not get into circular imports -from datalad.utils import with_pathsep as _with_sep # TODO: RF whenever merge conflict is not upon us from datalad.utils import ( - path_startswith, - path_is_subpath, ensure_unicode, getargspec, + path_is_subpath, + path_startswith, ) -from datalad.support.gitrepo import GitRepo -from datalad.support.exceptions import ( - CapturedException, -) -from datalad import cfg as dlcfg -from datalad.dochelpers import single_or_plural -from datalad.ui import ui -import datalad.support.ansi_colors as ac +from datalad.utils import \ + with_pathsep as \ + _with_sep # TODO: RF whenever merge conflict is not upon us anInterface = TypeVar('anInterface', bound='Interface') @@ -219,6 +218,7 @@ def _fx(res, **kwargs): def eval_results(wrapped): import warnings + from datalad.interface.base import eval_results as eval_results_moved warnings.warn("datalad.interface.utils.eval_results is obsolete. " "Use datalad.interface.base.eval_results instead", diff --git a/datalad/local/add_readme.py b/datalad/local/add_readme.py index 949bf87b8e..ed6c41723f 100644 --- a/datalad/local/add_readme.py +++ b/datalad/local/add_readme.py @@ -12,8 +12,10 @@ import logging -from datalad.interface.base import Interface -from datalad.interface.base import build_doc +from datalad.interface.base import ( + Interface, + build_doc, +) from datalad.support.annexrepo import AnnexRepo lgr = logging.getLogger('datalad.local.add_readme') @@ -29,15 +31,17 @@ class AddReadme(Interface): .gitattributes file exist. """ - from datalad.support.param import Parameter - from datalad.distribution.dataset import datasetmethod + from datalad.distribution.dataset import ( + EnsureDataset, + datasetmethod, + ) from datalad.interface.base import eval_results - from datalad.distribution.dataset import EnsureDataset from datalad.support.constraints import ( EnsureChoice, EnsureNone, EnsureStr, ) + from datalad.support.param import Parameter _params_ = dict( dataset=Parameter( @@ -67,9 +71,9 @@ def __call__(filename='README.md', *, dataset=None, existing='skip'): - from os.path import lexists - from os.path import join as opj from io import open + from os.path import join as opj + from os.path import lexists from datalad.distribution.dataset import require_dataset from datalad.utils import ensure_list diff --git a/datalad/local/addurls.py b/datalad/local/addurls.py index 522835560e..2221dc312a 100644 --- a/datalad/local/addurls.py +++ b/datalad/local/addurls.py @@ -1279,7 +1279,7 @@ class Addurls(Interface): WARNING: ONLY USE THIS OPTION IF YOU UNDERSTAND THE CONSEQUENCES. If the content of the URLs is not downloaded, then datalad will refuse to retrieve the contents with `datalad get ` by default - because the content of the URLs is not verified. Add + because the content of the URLs is not verified. Add `annex.security.allow-unverified-downloads = ACKTHPPT` to your git config to bypass the safety check. Underneath, this passes the `--fast` flag to `git annex addurl`."""), diff --git a/datalad/local/check_dates.py b/datalad/local/check_dates.py index d0a1d6cf02..f0f508f66e 100644 --- a/datalad/local/check_dates.py +++ b/datalad/local/check_dates.py @@ -13,8 +13,10 @@ import os import time -from datalad.interface.base import Interface -from datalad.interface.base import build_doc +from datalad.interface.base import ( + Interface, + build_doc, +) from datalad.interface.results import get_status_dict from datalad.support.exceptions import ( CapturedException, @@ -46,6 +48,7 @@ def _parse_date(date): msg="This package is required to parse non-timestamp dates") from calendar import timegm + # Note: datetime.timestamp isn't available in Python 2. try: timestamp = timegm(dateutil.parser.parse(date).utctimetuple()) @@ -66,9 +69,13 @@ class CheckDates(Interface): timestamps within files of the "git-annex" branch, and (3) the timestamps of annotated tags. """ - from datalad.interface.base import eval_results import datalad.support.ansi_colors as ac - from datalad.support.constraints import EnsureChoice, EnsureNone, EnsureStr + from datalad.interface.base import eval_results + from datalad.support.constraints import ( + EnsureChoice, + EnsureNone, + EnsureStr, + ) from datalad.support.param import Parameter result_renderer = "tailored" diff --git a/datalad/local/copy_file.py b/datalad/local/copy_file.py index 5d8d3aea2c..9fb3a64412 100644 --- a/datalad/local/copy_file.py +++ b/datalad/local/copy_file.py @@ -13,39 +13,36 @@ import logging import os.path as op -from shutil import copyfile import sys +from functools import lru_cache +from shutil import copyfile -from datalad.interface.base import Interface +from datalad.distribution.dataset import ( + Dataset, + EnsureDataset, + datasetmethod, + require_dataset, + resolve_path, +) from datalad.interface.base import ( + Interface, build_doc, eval_results, ) +from datalad.interface.common_opts import save_message_opt +from datalad.support.annexrepo import AnnexRepo from datalad.support.constraints import ( - EnsureStr, EnsureNone, + EnsureStr, ) -from datalad.interface.common_opts import save_message_opt -from datalad.support.param import Parameter -from datalad.distribution.dataset import ( - Dataset, - require_dataset, -) -from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import CapturedException +from datalad.support.param import Parameter from datalad.utils import ( + Path, ensure_list, get_dataset_root, - Path, ) -from datalad.distribution.dataset import ( - EnsureDataset, - datasetmethod, - resolve_path, -) -from functools import lru_cache - lgr = logging.getLogger('datalad.local.copy_file') diff --git a/datalad/local/export_archive.py b/datalad/local/export_archive.py index aae182dd4c..9da49dd556 100644 --- a/datalad/local/export_archive.py +++ b/datalad/local/export_archive.py @@ -10,23 +10,29 @@ __docformat__ = 'restructuredtext' -from datalad.interface.base import Interface -from datalad.interface.base import build_doc from pathlib import Path +from datalad.interface.base import ( + Interface, + build_doc, +) + + @build_doc class ExportArchive(Interface): """Export the content of a dataset as a TAR/ZIP archive. """ - from datalad.support.param import Parameter - from datalad.distribution.dataset import datasetmethod + from datalad.distribution.dataset import ( + EnsureDataset, + datasetmethod, + ) from datalad.interface.base import eval_results - from datalad.distribution.dataset import EnsureDataset from datalad.support.constraints import ( EnsureChoice, EnsureNone, EnsureStr, ) + from datalad.support.param import Parameter _params_ = dict( dataset=Parameter( @@ -76,15 +82,14 @@ def __call__(filename=None, archivetype='tar', compression='gz', missing_content='error'): + import logging import tarfile import zipfile from unittest.mock import patch from datalad.distribution.dataset import require_dataset - from datalad.utils import file_basename from datalad.support.annexrepo import AnnexRepo - - import logging + from datalad.utils import file_basename lgr = logging.getLogger('datalad.local.export_archive') dataset = require_dataset(dataset, check_installed=True, diff --git a/datalad/local/foreach_dataset.py b/datalad/local/foreach_dataset.py index 23418a977f..c849684da5 100644 --- a/datalad/local/foreach_dataset.py +++ b/datalad/local/foreach_dataset.py @@ -15,15 +15,16 @@ import logging import os.path as op import sys -from typing import Union - from argparse import REMAINDER from itertools import chain from tempfile import mkdtemp +from typing import Union -from datalad.cmd import NoCapture, StdOutErrCapture +from datalad.cmd import ( + NoCapture, + StdOutErrCapture, +) from datalad.core.local.run import normalize_command - from datalad.distribution.dataset import ( Dataset, EnsureDataset, @@ -56,9 +57,9 @@ no_subds_in_futures, ) from datalad.support.param import Parameter +from datalad.utils import SequenceFormatter +from datalad.utils import chpwd as chpwd_cm from datalad.utils import ( - SequenceFormatter, - chpwd as chpwd_cm, getpwd, nothing_cm, shortened_repr, diff --git a/datalad/local/gitcredential.py b/datalad/local/gitcredential.py index 6adba62cba..b9b77af929 100644 --- a/datalad/local/gitcredential.py +++ b/datalad/local/gitcredential.py @@ -11,13 +11,13 @@ """ from io import StringIO +from logging import getLogger from datalad.cmd import ( GitWitlessRunner, StdOutErrCapture, ) -from logging import getLogger lgr = getLogger('datalad.local.gitcredentials') @@ -154,5 +154,3 @@ def reject(self): ['git', 'credential', 'reject'], stdin=self._format_props() ) - - diff --git a/datalad/local/gitcredential_datalad.py b/datalad/local/gitcredential_datalad.py index 2a7626d98d..89f6feab1f 100644 --- a/datalad/local/gitcredential_datalad.py +++ b/datalad/local/gitcredential_datalad.py @@ -1,5 +1,6 @@ import sys +from datalad import ConfigManager from datalad.downloaders import ( GitCredential, UserPassword, @@ -9,7 +10,6 @@ Providers, ) from datalad.local.gitcredential import _credspec2dict -from datalad import ConfigManager git_credential_datalad_help = """\ Git credential interface to DataLad's credential management system. diff --git a/datalad/local/no_annex.py b/datalad/local/no_annex.py index 02b5dfa07e..4a17ddc65d 100644 --- a/datalad/local/no_annex.py +++ b/datalad/local/no_annex.py @@ -11,8 +11,10 @@ __docformat__ = 'restructuredtext' -from datalad.interface.base import Interface -from datalad.interface.base import build_doc +from datalad.interface.base import ( + Interface, + build_doc, +) @build_doc @@ -36,11 +38,13 @@ class NoAnnex(Interface): Note that this command works incrementally, hence any existing configuration (e.g. from a previous plugin run) is amended, not replaced. """ - from datalad.support.param import Parameter - from datalad.distribution.dataset import datasetmethod + from datalad.distribution.dataset import ( + EnsureDataset, + datasetmethod, + ) from datalad.interface.base import eval_results - from datalad.distribution.dataset import EnsureDataset from datalad.support.constraints import EnsureNone + from datalad.support.param import Parameter _params_ = dict( dataset=Parameter( @@ -76,10 +80,13 @@ class NoAnnex(Interface): # TODO*: make dataset, pattern into kwargs after *,? def __call__(dataset, pattern, ref_dir='.', makedirs=False): # could be extended to accept actual largefile expressions - from os.path import join as opj - from os.path import isabs - from os.path import exists from os import makedirs as makedirsfx + from os.path import ( + exists, + isabs, + ) + from os.path import join as opj + from datalad.distribution.dataset import require_dataset from datalad.support.annexrepo import AnnexRepo from datalad.utils import ensure_list diff --git a/datalad/local/remove.py b/datalad/local/remove.py index a8f5057cc0..adb39cd613 100644 --- a/datalad/local/remove.py +++ b/datalad/local/remove.py @@ -13,16 +13,16 @@ __docformat__ = 'restructuredtext' import logging -from os.path import lexists import warnings +from os.path import lexists from datalad.core.local.save import Save from datalad.core.local.status import get_paths_by_ds from datalad.distributed.drop import Drop from datalad.distribution.dataset import ( + EnsureDataset, datasetmethod, require_dataset, - EnsureDataset, ) from datalad.interface.base import ( Interface, diff --git a/datalad/local/rerun.py b/datalad/local/rerun.py index ed0383b6a5..cf6c14da88 100644 --- a/datalad/local/rerun.py +++ b/datalad/local/rerun.py @@ -37,9 +37,8 @@ build_doc, eval_results, ) -from datalad.interface.results import get_status_dict from datalad.interface.common_opts import jobs_opt - +from datalad.interface.results import get_status_dict from datalad.support.constraints import ( EnsureNone, EnsureStr, diff --git a/datalad/local/subdatasets.py b/datalad/local/subdatasets.py index 86c1779702..be3e6a8e1d 100644 --- a/datalad/local/subdatasets.py +++ b/datalad/local/subdatasets.py @@ -12,27 +12,22 @@ import logging -import re import os +import re import warnings -from datalad.interface.utils import generic_result_renderer +from datalad.distribution.dataset import ( + Dataset, + EnsureDataset, + datasetmethod, + require_dataset, + resolve_path, +) from datalad.interface.base import ( Interface, build_doc, eval_results, ) -from datalad.interface.results import get_status_dict -from datalad.support.constraints import ( - EnsureStr, - EnsureNone, - NoneDeprecated, -) -from datalad.support.param import Parameter -from datalad.support.exceptions import ( - CapturedException, - CommandError -) from datalad.interface.common_opts import ( contains, dataset_state, @@ -40,22 +35,24 @@ recursion_flag, recursion_limit, ) -from datalad.distribution.dataset import ( - Dataset, - require_dataset, +from datalad.interface.results import get_status_dict +from datalad.interface.utils import generic_result_renderer +from datalad.support.constraints import ( + EnsureNone, + EnsureStr, + NoneDeprecated, +) +from datalad.support.exceptions import ( + CapturedException, + CommandError, ) from datalad.support.gitrepo import GitRepo +from datalad.support.param import Parameter from datalad.utils import ( + Path, ensure_list, getpwd, partition, - Path, -) - -from datalad.distribution.dataset import ( - EnsureDataset, - datasetmethod, - resolve_path, ) lgr = logging.getLogger('datalad.local.subdatasets') diff --git a/datalad/local/tests/test_rerun.py b/datalad/local/tests/test_rerun.py index 888783ee24..37923e774b 100644 --- a/datalad/local/tests/test_rerun.py +++ b/datalad/local/tests/test_rerun.py @@ -17,9 +17,9 @@ import sys from io import StringIO from os import remove +from unittest.mock import patch import pytest -from unittest.mock import patch from datalad.api import ( install, diff --git a/datalad/local/tests/test_wtf.py b/datalad/local/tests/test_wtf.py index c7a362ae35..60e4925e38 100644 --- a/datalad/local/tests/test_wtf.py +++ b/datalad/local/tests/test_wtf.py @@ -21,6 +21,7 @@ _HIDDEN, SECTION_CALLABLES, ) +from datalad.support.external_versions import external_versions from datalad.tests.utils_pytest import ( DEFAULT_BRANCH, OBSCURE_FILENAME, @@ -37,8 +38,6 @@ ) from datalad.utils import ensure_unicode -from datalad.support.external_versions import external_versions - @with_tree({OBSCURE_FILENAME: {}}) def test_wtf(topdir=None): diff --git a/datalad/local/wtf.py b/datalad/local/wtf.py index 5815e320a5..791bbca058 100644 --- a/datalad/local/wtf.py +++ b/datalad/local/wtf.py @@ -16,28 +16,27 @@ import sys import tempfile import textwrap - from collections import defaultdict from functools import partial +from datalad import __version__ from datalad.dochelpers import get_docstring_split from datalad.interface.base import ( Interface, build_doc, ) -from datalad.utils import ( - ensure_unicode, - getpwd, - unlink, - Path, -) -from datalad.support.external_versions import external_versions from datalad.support.exceptions import ( CapturedException, CommandError, InvalidGitRepositoryError, ) -from datalad import __version__ +from datalad.support.external_versions import external_versions +from datalad.utils import ( + Path, + ensure_unicode, + getpwd, + unlink, +) lgr = logging.getLogger('datalad.local.wtf') @@ -65,6 +64,7 @@ def get_max_path_length(top_path=None, maxl=1000): if not top_path: top_path = getpwd() import random + from datalad import lgr from datalad.support import path prefix = path.join(top_path, "dl%d" % random.randint(1 ,100000)) @@ -124,6 +124,7 @@ def _describe_annex(): def _describe_system(): import platform as pl + from datalad import get_encoding_info from datalad.utils import get_linux_distribution try: @@ -212,9 +213,10 @@ def _describe_configuration(cfg, sensitive): def _describe_extensions(): infos = {} - from datalad.support.entrypoints import iter_entrypoints from importlib import import_module + from datalad.support.entrypoints import iter_entrypoints + for ename, emod, eload in iter_entrypoints('datalad.extensions'): info = {} infos[ename] = info @@ -247,8 +249,9 @@ def _describe_extensions(): def _describe_metadata_elements(group): infos = {} - from datalad.support.entrypoints import iter_entrypoints from importlib import import_module + + from datalad.support.entrypoints import iter_entrypoints if sys.version_info < (3, 10): # 3.10 is when it was no longer provisional from importlib_metadata import distribution @@ -393,11 +396,16 @@ class WTF(Interface): """ result_renderer = 'tailored' - from datalad.support.param import Parameter - from datalad.distribution.dataset import datasetmethod + from datalad.distribution.dataset import ( + EnsureDataset, + datasetmethod, + ) from datalad.interface.base import eval_results - from datalad.distribution.dataset import EnsureDataset - from datalad.support.constraints import EnsureNone, EnsureChoice + from datalad.support.constraints import ( + EnsureChoice, + EnsureNone, + ) + from datalad.support.param import Parameter _params_ = dict( dataset=Parameter( @@ -409,8 +417,8 @@ class WTF(Interface): sensitive=Parameter( args=("-s", "--sensitive",), constraints=EnsureChoice(None, 'some', 'all'), - doc="""if set to 'some' or 'all', it will display sections such as - config and metadata which could potentially contain sensitive + doc="""if set to 'some' or 'all', it will display sections such as + config and metadata which could potentially contain sensitive information (credentials, names, etc.). If 'some', the fields which are known to be sensitive will still be masked out"""), sections=Parameter( @@ -420,7 +428,7 @@ class WTF(Interface): metavar="SECTION", constraints=EnsureChoice(None, *sorted(SECTION_CALLABLES_GROUPPED) + ['*']), doc="""section to include. If not set - depends on flavor. - '*' could be used to force all sections. If there are subsections + '*' could be used to force all sections. If there are subsections like section.subsection available, then specifying just 'section' would select all subsections for that section. [CMD: This option can be given multiple times. CMD]"""), @@ -448,8 +456,8 @@ class WTF(Interface): @eval_results def __call__(*, dataset=None, sensitive=None, sections=None, flavor="full", decor=None, clipboard=None): from datalad.distribution.dataset import require_dataset - from datalad.support.exceptions import NoDatasetFound from datalad.interface.results import get_status_dict + from datalad.support.exceptions import NoDatasetFound ds = None try: @@ -478,8 +486,8 @@ def __call__(*, dataset=None, sensitive=None, sections=None, flavor="full", deco else: cfg = None - from datalad.ui import ui from datalad.support.external_versions import external_versions + from datalad.ui import ui infos = dict() res = get_status_dict( diff --git a/datalad/log.py b/datalad/log.py index 646b9cdeb7..32d71e0e46 100644 --- a/datalad/log.py +++ b/datalad/log.py @@ -8,23 +8,27 @@ # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Logging setup and utilities, including progress reporting""" -from contextlib import contextmanager -from functools import partial import inspect import logging +import logging.handlers import os -import sys import platform import random -import logging.handlers +import sys import warnings - -from os.path import basename, dirname - from collections import defaultdict +from contextlib import contextmanager +from functools import partial +from os.path import ( + basename, + dirname, +) -from .utils import is_interactive, optional_args from .support import ansi_colors as colors +from .utils import ( + is_interactive, + optional_args, +) __all__ = [ 'ColorFormatter', @@ -649,6 +653,7 @@ def get_initialized_logger(self, logtarget=None): names = self._get_config(names_filter, '') if names: import re + # add a filter which would catch those class LogFilter(object): """A log filter to filter based on the log target name(s)""" diff --git a/datalad/resources/procedures/cfg_text2git.py b/datalad/resources/procedures/cfg_text2git.py index 0563af1ef8..0ca7377a08 100644 --- a/datalad/resources/procedures/cfg_text2git.py +++ b/datalad/resources/procedures/cfg_text2git.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Procedure to configure Git annex to add text files directly to Git""" -import sys import os.path as op +import sys from datalad.distribution.dataset import require_dataset diff --git a/datalad/resources/procedures/cfg_yoda.py b/datalad/resources/procedures/cfg_yoda.py index 19fd5e4836..bff87ded98 100644 --- a/datalad/resources/procedures/cfg_yoda.py +++ b/datalad/resources/procedures/cfg_yoda.py @@ -5,8 +5,8 @@ `datalad create`. """ -import sys import os.path as op +import sys from datalad.distribution.dataset import require_dataset from datalad.utils import create_tree diff --git a/datalad/resources/procedures/tests/test_noannex.py b/datalad/resources/procedures/tests/test_noannex.py index f9b1144303..0113afd904 100644 --- a/datalad/resources/procedures/tests/test_noannex.py +++ b/datalad/resources/procedures/tests/test_noannex.py @@ -1,4 +1,4 @@ -#emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- +#emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- #ex: set sts=4 ts=4 sw=4 et: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # diff --git a/datalad/runner/gitrunner.py b/datalad/runner/gitrunner.py index 2192664125..33210c8dc8 100644 --- a/datalad/runner/gitrunner.py +++ b/datalad/runner/gitrunner.py @@ -12,7 +12,6 @@ import logging import os import os.path as op - from typing import Optional from datalad.dochelpers import borrowdoc @@ -20,12 +19,12 @@ generate_file_chunks, make_tempfile, ) + from .runner import ( GeneratorMixIn, WitlessRunner, ) - lgr = logging.getLogger('datalad.runner.gitrunner') # We use custom ssh runner while interacting with git diff --git a/datalad/runner/nonasyncrunner.py b/datalad/runner/nonasyncrunner.py index 6895d91fb0..502a81b480 100644 --- a/datalad/runner/nonasyncrunner.py +++ b/datalad/runner/nonasyncrunner.py @@ -45,7 +45,6 @@ _try_close, ) - __docformat__ = 'restructuredtext' lgr = logging.getLogger("datalad.runner.nonasyncrunner") diff --git a/datalad/runner/runner.py b/datalad/runner/runner.py index 32c8074ee3..5d3564cb8b 100644 --- a/datalad/runner/runner.py +++ b/datalad/runner/runner.py @@ -18,15 +18,16 @@ cast, ) -from .protocol import WitlessProtocol from .coreprotocols import NoCapture from .exception import CommandError from .nonasyncrunner import ( ThreadedRunner, _ResultGenerator, ) -from .protocol import GeneratorMixIn - +from .protocol import ( + GeneratorMixIn, + WitlessProtocol, +) lgr = logging.getLogger('datalad.runner.runner') diff --git a/datalad/support/annexrepo.py b/datalad/support/annexrepo.py index a92ac00f61..04a8d5d709 100644 --- a/datalad/support/annexrepo.py +++ b/datalad/support/annexrepo.py @@ -58,7 +58,6 @@ AssemblingDecoderMixIn, LineSplitter, ) - from datalad.support.annex_utils import ( _fake_json_for_non_existing, _get_non_existing_from_annex_output, diff --git a/datalad/support/archive_utils_7z.py b/datalad/support/archive_utils_7z.py index b7fb50c696..83f55bda0a 100644 --- a/datalad/support/archive_utils_7z.py +++ b/datalad/support/archive_utils_7z.py @@ -10,25 +10,25 @@ from datalad.support.external_versions import external_versions + external_versions.check( "cmd:7z", msg='The 7z binary (7-Zip) is required for archive handling, but is missing. ' "Setting the config flag 'datalad.runtime.use-patool' enables an " "alternative implementation that may not need 7z.") +import logging + from datalad.utils import ( Path, join_cmdline, quote_cmdlinearg, ) -import logging lgr = logging.getLogger('datalad.support.archive_utils_7z') -from datalad.cmd import ( - WitlessRunner as Runner, - KillOutput, -) +from datalad.cmd import KillOutput +from datalad.cmd import WitlessRunner as Runner def _normalize_fname_suffixes(suffixes): diff --git a/datalad/support/archive_utils_patool.py b/datalad/support/archive_utils_patool.py index 9c106cb9db..4a5eb5e6b5 100644 --- a/datalad/support/archive_utils_patool.py +++ b/datalad/support/archive_utils_patool.py @@ -9,28 +9,46 @@ """patool based implementation for datalad.support.archives utilities""" import patoolib + from .external_versions import external_versions + # There were issues, so let's stay consistently with recent version assert(external_versions["patoolib"] >= "1.7") +import logging import os -from .exceptions import MissingExternalDependency -from .path import ( - basename, - join as opj, -) from datalad.utils import ( - ensure_bytes, chpwd, + ensure_bytes, ) -import logging +from .exceptions import MissingExternalDependency +from .path import basename +from .path import join as opj + lgr = logging.getLogger('datalad.support.archive_utils_patool') # Monkey-patch patoolib's logging, so it logs coherently with the rest of # datalad import patoolib.util + +from datalad.cmd import ( + StdOutErrCapture, + WitlessRunner, +) + +# we need to decorate patool.util.run +# because otherwise it just lets processes to spit out everything to std and we +# do want to use it at "verbosity>=0" so we could get idea on what is going on. +# And I don't want to mock for every invocation +from ..support.exceptions import CommandError +from ..utils import ( + ensure_unicode, + on_windows, + swallow_outputs, +) + # # Seems have managed with swallow_outputs # @@ -47,19 +65,7 @@ # patoolib.util.log_error = _patool_log_error # patoolib.util.log_internal_error = _patool_log_error -# we need to decorate patool.util.run -# because otherwise it just lets processes to spit out everything to std and we -# do want to use it at "verbosity>=0" so we could get idea on what is going on. -# And I don't want to mock for every invocation -from ..support.exceptions import CommandError -from ..utils import swallow_outputs -from datalad.cmd import ( - WitlessRunner, - StdOutErrCapture, -) -from ..utils import ensure_unicode -from ..utils import on_windows _runner = WitlessRunner() diff --git a/datalad/support/cache.py b/datalad/support/cache.py index 387cc703ca..3a90cce3d3 100644 --- a/datalad/support/cache.py +++ b/datalad/support/cache.py @@ -10,7 +10,6 @@ """ from collections import OrderedDict - from functools import lru_cache diff --git a/datalad/support/due.py b/datalad/support/due.py index f5ecd8d9c3..a29bbcb36f 100644 --- a/datalad/support/due.py +++ b/datalad/support/due.py @@ -52,7 +52,13 @@ def _donothing_func(*args, **kwargs): try: - from duecredit import due, BibTeX, Doi, Url, Text + from duecredit import ( + BibTeX, + Doi, + Text, + Url, + due, + ) if 'due' in locals() and not hasattr(due, 'cite'): raise RuntimeError( "Imported due lacks .cite. DueCredit is now disabled") diff --git a/datalad/support/due_utils.py b/datalad/support/due_utils.py index a9d8cb5c6c..cedee850f4 100644 --- a/datalad/support/due_utils.py +++ b/datalad/support/due_utils.py @@ -1,15 +1,24 @@ -#emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- +#emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- #ex: set sts=4 ts=4 sw=4 et: """ Support functionality for using DueCredit """ -# Note Text was added/exposed only since DueCredit 0.6.5 -from .due import due, Doi, Text -from ..utils import never_fail, swallow_logs +import logging + from datalad.support.exceptions import CapturedException -import logging +from ..utils import ( + never_fail, + swallow_logs, +) +# Note Text was added/exposed only since DueCredit 0.6.5 +from .due import ( + Doi, + Text, + due, +) + lgr = logging.getLogger('datalad.duecredit') @@ -69,7 +78,7 @@ def duecredit_dataset(dataset): if not isinstance(res, dict): lgr.debug("Got record which is not a dict, no duecredit for now") return - + metadata = res.get('metadata', {}) # Descend following the dots -- isn't there a helper already - TODO? diff --git a/datalad/support/extensions.py b/datalad/support/extensions.py index 58a7d595a8..bce9fc116c 100644 --- a/datalad/support/extensions.py +++ b/datalad/support/extensions.py @@ -9,11 +9,12 @@ """Support functionality for extension development""" import logging + from datalad.interface.common_cfg import ( - _NotGiven, _ConfigDefinition, - definitions as _definitions, + _NotGiven, ) +from datalad.interface.common_cfg import definitions as _definitions __all__ = ['register_config', 'has_config'] diff --git a/datalad/support/gitrepo.py b/datalad/support/gitrepo.py index 478f6dc878..5904bebb8e 100644 --- a/datalad/support/gitrepo.py +++ b/datalad/support/gitrepo.py @@ -1006,13 +1006,13 @@ def clone(cls, url: str, path: str, *args: Any, clone_options: Optional[list[str url_ri = RI(url) if not isinstance(url, RI) else url if on_windows: # When we're cloning from a local path on Windows, the URL at - # this point is platform-specific (e.g., "..\\origin"). According + # this point is platform-specific (e.g., "..\\origin"). According # to Git clone's manpage, clone urls can't have backslashes. - # While Git does manage to clone a URL with backslashes, + # While Git does manage to clone a URL with backslashes, # in the case of subdatasets cloned from relative paths it nevertheless # messed up the resulting remote url, resulting in a mix of - # front and backslashes (see also gh-7180): - # 'C:/Users/adina/AppData/Local/Temp/datalad_temp_frvczceh/ds/..\\origin' + # front and backslashes (see also gh-7180): + # 'C:/Users/adina/AppData/Local/Temp/datalad_temp_frvczceh/ds/..\\origin' # Therefore, we're turning it to Posix now. if isinstance(url_ri, PathRI): url = Path(url).as_posix() diff --git a/datalad/support/json_py.py b/datalad/support/json_py.py index 16254166d7..4384854f5a 100644 --- a/datalad/support/json_py.py +++ b/datalad/support/json_py.py @@ -10,21 +10,20 @@ """ -import io import codecs +import io +import json +import os +import os.path as op +from os import makedirs from os.path import ( dirname, exists, lexists, ) -from os import makedirs -import os -import os.path as op -import json from datalad.support.exceptions import CapturedException - # produce relatively compact, but also diff-friendly format json_dump_kwargs = dict( indent=0, diff --git a/datalad/support/keyring_.py b/datalad/support/keyring_.py index 9c1ac94e10..efe0ad8242 100644 --- a/datalad/support/keyring_.py +++ b/datalad/support/keyring_.py @@ -9,8 +9,9 @@ """Adapters and decorators for keyrings """ -import os import logging +import os + lgr = logging.getLogger('datalad.support.keyring') @@ -36,8 +37,9 @@ def _keyring(self): if self.__keyring_mod is None: # Setup logging for keyring if we are debugging, although keyring's logging # is quite scarce ATM - from datalad.log import lgr import logging + + from datalad.log import lgr lgr_level = lgr.getEffectiveLevel() if lgr_level < logging.DEBUG: keyring_lgr = logging.getLogger('keyring') @@ -49,6 +51,7 @@ def _keyring(self): if self.__keyring is None: from datalad.log import lgr + # we use module bound interfaces whenever we were not provided a dedicated # backend self.__keyring = self.__keyring_mod diff --git a/datalad/support/locking.py b/datalad/support/locking.py index 0d0e7e34ec..f946312306 100644 --- a/datalad/support/locking.py +++ b/datalad/support/locking.py @@ -1,19 +1,20 @@ +import logging +from contextlib import contextmanager + from fasteners import ( InterProcessLock, try_lock, ) -from contextlib import contextmanager -from .path import exists +from datalad.support.exceptions import CapturedException + from ..utils import ( ensure_unicode, get_open_files, unlink, ) -from datalad.support.exceptions import CapturedException - +from .path import exists -import logging lgr = logging.getLogger('datalad.locking') diff --git a/datalad/support/nda_.py b/datalad/support/nda_.py index f23c1459c4..effd7f3135 100644 --- a/datalad/support/nda_.py +++ b/datalad/support/nda_.py @@ -14,10 +14,11 @@ __docformat__ = 'restructuredtext' +from logging import getLogger + from datalad import cfg from datalad.downloaders.providers import Providers -from logging import getLogger lgr = getLogger('datalad.support.nda') DEFAULT_SERVER = 'mindarvpc.cqahbwk3l1mb.us-east-1.rds.amazonaws.com' @@ -61,7 +62,7 @@ def get_oracle_db( if not isinstance(credential, dict): credential = credential() - import cx_Oracle # you must have the beast if you want to access the dark side + import cx_Oracle # you must have the beast if you want to access the dark side dsnStr = cx_Oracle.makedsn(dbserver, port, sid) db = cx_Oracle.connect(user=credential['user'], password=credential['password'], diff --git a/datalad/support/network.py b/datalad/support/network.py index 0ce470ea02..071290a3a5 100644 --- a/datalad/support/network.py +++ b/datalad/support/network.py @@ -29,10 +29,10 @@ ParseResult, parse_qsl, ) +from urllib.parse import quote from urllib.parse import quote as urlquote from urllib.parse import unquote as urlunquote from urllib.parse import ( - quote, urlencode, urljoin, urlparse, diff --git a/datalad/support/parallel.py b/datalad/support/parallel.py index 5bef71429b..8b4f01d828 100644 --- a/datalad/support/parallel.py +++ b/datalad/support/parallel.py @@ -14,20 +14,23 @@ import concurrent.futures import inspect +import logging import sys import time import uuid - from collections import defaultdict -from queue import Queue, Empty +from queue import ( + Empty, + Queue, +) from threading import Thread -from . import ansi_colors as colors +from datalad.support.exceptions import CapturedException + from ..log import log_progress from ..utils import path_is_subpath -from datalad.support.exceptions import CapturedException +from . import ansi_colors as colors -import logging lgr = logging.getLogger('datalad.parallel') @@ -250,6 +253,7 @@ def get_effective_jobs(cls, jobs): """ if jobs in (None, "auto"): from datalad import cfg + # ATM there is no "auto" for this operation, so in both auto and None # just consult max-jobs which can only be an int ATM. # "auto" could be for some auto-scaling based on a single future time diff --git a/datalad/support/param.py b/datalad/support/param.py index 2c8a7c61e6..0793bfb782 100644 --- a/datalad/support/param.py +++ b/datalad/support/param.py @@ -10,9 +10,10 @@ __docformat__ = 'restructuredtext' +import argparse import re import textwrap -import argparse + from datalad.utils import getargspec from .constraints import expand_constraint_spec diff --git a/datalad/support/repodates.py b/datalad/support/repodates.py index 4fa977d48b..2b2ffbb290 100644 --- a/datalad/support/repodates.py +++ b/datalad/support/repodates.py @@ -14,7 +14,6 @@ import re import time - from datalad.log import log_progress from datalad.support.exceptions import CommandError from datalad.support.gitrepo import GitRepo diff --git a/datalad/support/s3.py b/datalad/support/s3.py index caa58872e7..0890c96c56 100644 --- a/datalad/support/s3.py +++ b/datalad/support/s3.py @@ -15,18 +15,21 @@ __docformat__ = 'restructuredtext' +import logging import mimetypes -from pathlib import PurePath import re -from urllib.request import urlopen, Request +from pathlib import PurePath +from urllib.request import ( + Request, + urlopen, +) import boto3 from botocore.exceptions import ClientError +import datalad.log # Just to have lgr setup happen this one used a script from datalad.support.network import URL -import logging -import datalad.log # Just to have lgr setup happen this one used a script lgr = logging.getLogger('datalad.s3') # TODO: should become a config option and managed along with the rest diff --git a/datalad/support/sshconnector.py b/datalad/support/sshconnector.py index fe6847fec3..410fc57392 100644 --- a/datalad/support/sshconnector.py +++ b/datalad/support/sshconnector.py @@ -13,44 +13,42 @@ git calls to a ssh remote without the need to reauthenticate. """ -import fasteners -import os import logging -from socket import gethostname -from hashlib import md5 -from subprocess import Popen +import os import tempfile import threading +from hashlib import md5 +from socket import gethostname +from subprocess import Popen +import fasteners -# importing the quote function here so it can always be imported from this -# module -# this used to be shlex.quote(), but is now a cross-platform helper -from datalad.utils import quote_cmdlinearg as sh_quote - -# !!! Do not import network here -- delay import, allows to shave off 50ms or so -# on initial import datalad time -# from datalad.support.network import RI, is_ssh - +from datalad.cmd import ( + NoCapture, + StdOutErrCapture, + WitlessRunner, +) from datalad.support.exceptions import ( CapturedException, CommandError, ConnectionOpenFailedError, ) -from datalad.support.external_versions import ( - external_versions, -) +from datalad.support.external_versions import external_versions +# importing the quote function here so it can always be imported from this +# module +# this used to be shlex.quote(), but is now a cross-platform helper from datalad.utils import ( - auto_repr, Path, + auto_repr, ensure_list, on_windows, ) -from datalad.cmd import ( - NoCapture, - StdOutErrCapture, - WitlessRunner, -) +from datalad.utils import quote_cmdlinearg as sh_quote + +# !!! Do not import network here -- delay import, allows to shave off 50ms or so +# on initial import datalad time +# from datalad.support.network import RI, is_ssh + lgr = logging.getLogger('datalad.support.sshconnector') @@ -136,7 +134,10 @@ def __init__(self, sshri, identity_file=None, self._ssh_executable = None self._ssh_version = None - from datalad.support.network import SSHRI, is_ssh + from datalad.support.network import ( + SSHRI, + is_ssh, + ) if not is_ssh(sshri): raise ValueError( "Non-SSH resource identifiers are not supported for SSH " @@ -226,6 +227,7 @@ def ssh_version(self): def _adjust_cmd_for_bundle_execution(self, cmd): from datalad import cfg + # locate annex and set the bundled vs. system Git machinery in motion if self._use_remote_annex_bundle \ or cfg.obtain('datalad.ssh.try-use-annex-bundled-git'): @@ -652,7 +654,10 @@ def get_connection(self, url, use_remote_annex_bundle=None, force_ip=False): def _prep_connection_args(self, url): # parse url: - from datalad.support.network import RI, is_ssh + from datalad.support.network import ( + RI, + is_ssh, + ) if isinstance(url, RI): sshri = url else: @@ -828,6 +833,7 @@ def close(self, allow_fail=True, ctrl_path=None): # retain backward compat with 0.13.4 and earlier # should be ok since cfg already defined by the time this one is imported from .. import cfg + if cfg.obtain('datalad.ssh.multiplex-connections'): SSHManager = MultiplexSSHManager SSHConnection = MultiplexSSHConnection diff --git a/datalad/support/sshrun.py b/datalad/support/sshrun.py index c2685d5d75..1c0600aeb7 100644 --- a/datalad/support/sshrun.py +++ b/datalad/support/sshrun.py @@ -20,13 +20,14 @@ import sys import tempfile +from datalad import ssh_manager +from datalad.interface.base import ( + Interface, + build_doc, +) from datalad.support.param import Parameter -from datalad.interface.base import Interface -from datalad.interface.base import build_doc from datalad.utils import split_cmdline -from datalad import ssh_manager - lgr = logging.getLogger('datalad.sshrun') diff --git a/datalad/support/tests/test_annexrepo.py b/datalad/support/tests/test_annexrepo.py index d9623f99d1..fc006f43ff 100644 --- a/datalad/support/tests/test_annexrepo.py +++ b/datalad/support/tests/test_annexrepo.py @@ -137,7 +137,6 @@ unlink, ) - _GIT_ANNEX_VERSIONS_INFO = AnnexRepo.check_repository_versions() diff --git a/datalad/support/tests/test_extensions.py b/datalad/support/tests/test_extensions.py index af2222cf94..46d4d22c08 100644 --- a/datalad/support/tests/test_extensions.py +++ b/datalad/support/tests/test_extensions.py @@ -65,5 +65,3 @@ def test_register_config(): nkeys = len(df) df.update(funky='seven') eq_(len(df), nkeys + 1) - - diff --git a/datalad/support/tests/test_external_versions.py b/datalad/support/tests/test_external_versions.py index 6b69d3f7d8..307180116e 100644 --- a/datalad/support/tests/test_external_versions.py +++ b/datalad/support/tests/test_external_versions.py @@ -8,11 +8,15 @@ # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import logging -import pytest from os import linesep +import pytest + from datalad import __version__ -from datalad.cmd import StdOutErrCapture, WitlessRunner +from datalad.cmd import ( + StdOutErrCapture, + WitlessRunner, +) from datalad.support.annexrepo import AnnexRepo from datalad.support.exceptions import ( CommandError, diff --git a/datalad/support/tests/test_fileinfo.py b/datalad/support/tests/test_fileinfo.py index 7391300446..2592df72ea 100644 --- a/datalad/support/tests/test_fileinfo.py +++ b/datalad/support/tests/test_fileinfo.py @@ -28,8 +28,8 @@ known_failure_githubci_win, on_nfs, on_travis, - slow, skip_if, + slow, with_tempfile, with_tree, ) diff --git a/datalad/support/tests/test_locking.py b/datalad/support/tests/test_locking.py index 055dca8f79..bf80cd01f5 100644 --- a/datalad/support/tests/test_locking.py +++ b/datalad/support/tests/test_locking.py @@ -1,4 +1,4 @@ -#emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- +#emacs: -*- mode: python-mode; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*- #ex: set sts=4 ts=4 sw=4 et: # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## # diff --git a/datalad/support/tests/test_network.py b/datalad/support/tests/test_network.py index a64df33fd3..918ed02eea 100644 --- a/datalad/support/tests/test_network.py +++ b/datalad/support/tests/test_network.py @@ -34,14 +34,14 @@ is_ssh, is_url, iso8601_to_epoch, - local_path_representation, local_path2url_path, + local_path_representation, local_url_path_representation, parse_url_opts, quote_path, same_website, - urlquote, url_path2local_path, + urlquote, ) from datalad.tests.utils_pytest import ( OBSCURE_FILENAME, diff --git a/datalad/support/tests/test_path.py b/datalad/support/tests/test_path.py index cb5de16707..012f2ad823 100644 --- a/datalad/support/tests/test_path.py +++ b/datalad/support/tests/test_path.py @@ -10,6 +10,8 @@ import os from pathlib import PurePosixPath +import pytest + from ...tests.utils_pytest import ( SkipTest, assert_raises, @@ -24,14 +26,12 @@ from ..path import ( abspath, curdir, - get_parent_paths, get_filtered_paths_, + get_parent_paths, robust_abspath, split_ext, ) -import pytest - @with_tempfile(mkdir=True) def test_robust_abspath(tdir=None): @@ -146,4 +146,4 @@ def gfp(*args, **kwargs): assert_raises(ValueError, gfp, ['/a'], []) assert_raises(ValueError, gfp, [PurePosixPath('/a')], []) assert_raises(ValueError, gfp, ['a'], ['/a']) - assert_raises(ValueError, gfp, ['../a'], ['a']) \ No newline at end of file + assert_raises(ValueError, gfp, ['../a'], ['a']) diff --git a/datalad/support/tests/test_sshrun.py b/datalad/support/tests/test_sshrun.py index e06c3894c6..510f3f7df0 100644 --- a/datalad/support/tests/test_sshrun.py +++ b/datalad/support/tests/test_sshrun.py @@ -8,11 +8,13 @@ # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## import sys -from io import StringIO, UnsupportedOperation - -import pytest +from io import ( + StringIO, + UnsupportedOperation, +) from unittest.mock import patch +import pytest from datalad.api import sshrun from datalad.cli.main import main diff --git a/datalad/support/third/loris_token_generator.py b/datalad/support/third/loris_token_generator.py index 3118f5b861..1f2dc2430d 100644 --- a/datalad/support/third/loris_token_generator.py +++ b/datalad/support/third/loris_token_generator.py @@ -6,18 +6,15 @@ # copyright and license terms. # # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### # -import sys import json - +import sys +from urllib.error import HTTPError from urllib.request import ( Request, urlopen, ) -from urllib.error import HTTPError -from datalad.support.exceptions import ( - AccessDeniedError, -) +from datalad.support.exceptions import AccessDeniedError from datalad.utils import ensure_unicode @@ -48,4 +45,3 @@ def generate_token(self, user=None, password=None): str_response = ensure_unicode(response.read()) data = json.loads(str_response) return data["token"] - diff --git a/datalad/support/third/nda_aws_token_generator.py b/datalad/support/third/nda_aws_token_generator.py index d54b91643c..a10b0d3d8b 100644 --- a/datalad/support/third/nda_aws_token_generator.py +++ b/datalad/support/third/nda_aws_token_generator.py @@ -7,9 +7,8 @@ import binascii import hashlib import logging -import xml.etree.ElementTree as etree - import sys +import xml.etree.ElementTree as etree if sys.version_info[0] == 2: import urllib2 as urllib_request diff --git a/datalad/support/vcr_.py b/datalad/support/vcr_.py index 129d833c5a..c2f2dec18b 100644 --- a/datalad/support/vcr_.py +++ b/datalad/support/vcr_.py @@ -10,13 +10,12 @@ """ import logging - +from contextlib import contextmanager from functools import wraps from os.path import isabs -from contextlib import contextmanager -from datalad.utils import Path from datalad.support.exceptions import CapturedException +from datalad.utils import Path lgr = logging.getLogger("datalad.support.vcr") @@ -30,16 +29,20 @@ def _get_cassette_path(path): try: # TEMP: Just to overcome problem with testing on jessie with older requests # https://github.com/kevin1024/vcrpy/issues/215 - import vcr.patch as _vcrp import requests as _ + import vcr.patch as _vcrp try: - from requests.packages.urllib3.connectionpool import HTTPConnection as _a, VerifiedHTTPSConnection as _b + from requests.packages.urllib3.connectionpool import \ + HTTPConnection as _a + from requests.packages.urllib3.connectionpool import \ + VerifiedHTTPSConnection as _b except ImportError: def returnnothing(*args, **kwargs): return() _vcrp.CassettePatcherBuilder._requests = returnnothing - from vcr import use_cassette as _use_cassette, VCR as _VCR + from vcr import VCR as _VCR + from vcr import use_cassette as _use_cassette def use_cassette(path, return_body=None, skip_if_no_vcr=False, **kwargs): """Adapter so we could create/use custom use_cassette with custom parameters diff --git a/datalad/tests/__init__.py b/datalad/tests/__init__.py index 60df60b014..cf0fd43182 100644 --- a/datalad/tests/__init__.py +++ b/datalad/tests/__init__.py @@ -21,4 +21,3 @@ # Give a custom template so we could hunt them down easily tempfile.template = os.path.join(tempfile.gettempdir(), 'tmp-page2annex') - diff --git a/datalad/tests/ca/README b/datalad/tests/ca/README index 68b2af4b0c..a9b5cbf5df 100644 --- a/datalad/tests/ca/README +++ b/datalad/tests/ca/README @@ -29,5 +29,5 @@ cat ca-root.pem certificate-pub.pem > ca_bundle.pem # check if certificate is considered valid openssl verify -trusted ca_bundle.pem certificate-pub.pem -# potentially +# potentially #export REQUESTS_CA_BUNDLE=/path/to/ca_bundle.pem diff --git a/datalad/tests/test_base.py b/datalad/tests/test_base.py index 30b782186c..5065058b0c 100644 --- a/datalad/tests/test_base.py +++ b/datalad/tests/test_base.py @@ -12,7 +12,10 @@ import sys from unittest.mock import patch -from datalad.cmd import WitlessRunner, StdOutErrCapture +from datalad.cmd import ( + StdOutErrCapture, + WitlessRunner, +) from datalad.tests.utils_pytest import ( SkipTest, assert_in, diff --git a/datalad/tests/test_cmd.py b/datalad/tests/test_cmd.py index 05f4e44944..437486bc2b 100644 --- a/datalad/tests/test_cmd.py +++ b/datalad/tests/test_cmd.py @@ -16,9 +16,9 @@ from datalad.cmd import ( BatchedCommand, + BatchedCommandError, readline_rstripped, ) -from datalad.cmd import BatchedCommandError from datalad.runner.tests.utils import py2cmd from datalad.tests.utils_pytest import ( assert_equal, diff --git a/datalad/tests/test_tests_utils_pytest.py b/datalad/tests/test_tests_utils_pytest.py index 7eebe9b9e1..deb727084e 100644 --- a/datalad/tests/test_tests_utils_pytest.py +++ b/datalad/tests/test_tests_utils_pytest.py @@ -95,8 +95,6 @@ getpwd, ) -import pytest - # # Test with_tempfile, especially nested invocations # @@ -263,7 +261,7 @@ def test_ok_symlink_helpers(tmpfile=None): with open(tmpfile, 'w') as tf: tf.write('test text') - + # tmpfile is still not a symlink here assert_raises(AssertionError, ok_symlink, tmpfile) assert_raises(AssertionError, ok_good_symlink, tmpfile) @@ -694,4 +692,4 @@ def raise_TypeError(): raise_TypeError() assert_raises((ValueError, TypeError), raise_TypeError) - assert_raises((ValueError, TypeError), raise_ValueError) \ No newline at end of file + assert_raises((ValueError, TypeError), raise_ValueError) diff --git a/datalad/tests/utils_pytest.py b/datalad/tests/utils_pytest.py index 547593f83a..2814c7cfaf 100644 --- a/datalad/tests/utils_pytest.py +++ b/datalad/tests/utils_pytest.py @@ -28,15 +28,13 @@ import datalad.utils as ut from datalad import cfg as dl_cfg +from datalad import utils from datalad.cmd import ( StdOutErrCapture, WitlessRunner, ) - -from datalad import utils from datalad.consts import ARCHIVES_TEMP_DIR from datalad.dochelpers import borrowkwargs - from datalad.support.external_versions import ( external_versions, get_rsync_version, @@ -47,7 +45,6 @@ # TODO this must go from datalad.utils import * - # temp paths used by clones _TEMP_PATHS_CLONES = set() @@ -2052,7 +2049,7 @@ def set_annex_version(version): def integration(f): """Mark test as an "integration" test which generally is not needed to be run - + Generally tend to be slower. Should be used in combination with @slow and @turtle if that is the case. """ diff --git a/datalad/tests/utils_testdatasets.py b/datalad/tests/utils_testdatasets.py index ccd14ca215..b554d70d7f 100644 --- a/datalad/tests/utils_testdatasets.py +++ b/datalad/tests/utils_testdatasets.py @@ -9,8 +9,10 @@ from os.path import join as opj + from datalad.distribution.dataset import Dataset + def _make_dataset_hierarchy(path): origin = Dataset(path).create() origin_sub1 = origin.create('sub1') @@ -36,4 +38,4 @@ def _mk_submodule_annex(path, fname, fcontent): s2 = origds.create('2', **ca) (s2.pathobj / fname).write_text(fcontent) origds.save(recursive=True, **ca) - return origds \ No newline at end of file + return origds diff --git a/datalad/tests/utils_testrepos.py b/datalad/tests/utils_testrepos.py index b0e201eb71..490a2c082e 100644 --- a/datalad/tests/utils_testrepos.py +++ b/datalad/tests/utils_testrepos.py @@ -13,9 +13,7 @@ ABCMeta, abstractmethod, ) -from os.path import ( - exists, -) +from os.path import exists from os.path import join as opj from datalad import cfg as dl_cfg diff --git a/datalad/ui/__init__.py b/datalad/ui/__init__.py index 72c7790a93..6dea8f2817 100644 --- a/datalad/ui/__init__.py +++ b/datalad/ui/__init__.py @@ -13,22 +13,23 @@ __docformat__ = 'restructuredtext' from logging import getLogger + lgr = getLogger('datalad.ui') lgr.log(5, "Starting importing ui") +from ..utils import ( + get_ipython_shell, + is_interactive, +) from .dialog import ( ConsoleLog, DialogUI, IPythonUI, + QuietConsoleLog, + SilentConsoleLog, UnderAnnexUI, UnderTestsUI, - SilentConsoleLog, - QuietConsoleLog, -) -from ..utils import ( - is_interactive, - get_ipython_shell, ) KNOWN_BACKENDS = { diff --git a/datalad/ui/base.py b/datalad/ui/base.py index 5ba85e7c47..4dc8d01040 100644 --- a/datalad/ui/base.py +++ b/datalad/ui/base.py @@ -12,7 +12,10 @@ __docformat__ = 'restructuredtext' -from abc import ABCMeta, abstractmethod +from abc import ( + ABCMeta, + abstractmethod, +) from ..utils import auto_repr diff --git a/datalad/ui/dialog.py b/datalad/ui/dialog.py index c181b71343..9b93d523f9 100644 --- a/datalad/ui/dialog.py +++ b/datalad/ui/dialog.py @@ -13,26 +13,28 @@ __docformat__ = 'restructuredtext' from logging import getLogger + lgr = getLogger('datalad.ui.dialog') lgr.log(5, "Starting importing ui.dialog") +import getpass import os import sys import time - -import getpass - #!!! OPT adds >100ms to import time!!! # from unittest.mock import patch from collections import deque from copy import copy -from ..utils import auto_repr -from ..utils import on_windows +from datalad.support.exceptions import CapturedException + +from ..utils import ( + auto_repr, + on_windows, +) from .base import InteractiveUI from .utils import can_prompt -from datalad.support.exceptions import CapturedException # Example APIs which might be useful to look for "inspiration" # man debconf-devel diff --git a/datalad/ui/progressbars.py b/datalad/ui/progressbars.py index 3d855e2f82..f466265f23 100644 --- a/datalad/ui/progressbars.py +++ b/datalad/ui/progressbars.py @@ -11,10 +11,11 @@ Should not be imported until we know that interface needs it """ -import humanize import sys import time +import humanize + from .. import lgr # @@ -190,6 +191,7 @@ def finish(self, partial=False): try: from tqdm import tqdm + from datalad.utils import updated class tqdmProgressBar(ProgressBarBase): diff --git a/datalad/ui/utils.py b/datalad/ui/utils.py index b1d002f740..e277f08e3d 100644 --- a/datalad/ui/utils.py +++ b/datalad/ui/utils.py @@ -8,9 +8,10 @@ # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## """Various utils oriented to UI""" +import struct + from datalad.support import ansi_colors from datalad.utils import on_windows -import struct # origin http://stackoverflow.com/a/3010495/1265472 @@ -18,7 +19,10 @@ def get_terminal_size(): """Return current terminal size""" if on_windows: try: - from ctypes import windll, create_string_buffer + from ctypes import ( + create_string_buffer, + windll, + ) # stdin handle is -10 # stdout handle is -11 diff --git a/docs/casts/basic_search.sh b/docs/casts/basic_search.sh index e565f3d02f..e864dc4017 100644 --- a/docs/casts/basic_search.sh +++ b/docs/casts/basic_search.sh @@ -30,4 +30,4 @@ say "and get data we are interested in, e.g. all the anatomicals within those in run "datalad get -J4 openfmri/ds000*/sub-*/anat" run "datalad ls -Lr . | grep -v 'not installed'" -say "Now it is your turn to find some interesting datasets for yourself!" \ No newline at end of file +say "Now it is your turn to find some interesting datasets for yourself!" diff --git a/docs/casts/reproducible_analysis.sh b/docs/casts/reproducible_analysis.sh index 44907aa45c..b3d8f9f701 100644 --- a/docs/casts/reproducible_analysis.sh +++ b/docs/casts/reproducible_analysis.sh @@ -35,7 +35,7 @@ for nifti in \\\$@; do mkdir -p \\\$subdir echo \"Processing \\\$nifti\" bet \\\$nifti \\\$subdir/anat -m - count=\\\$((count + 1)) + count=\\\$((count + 1)) done EOT" diff --git a/docs/design.rst b/docs/design.rst index ccc2384a02..e8a5350449 100644 --- a/docs/design.rst +++ b/docs/design.rst @@ -8,8 +8,8 @@ Global portion of the config [dataset] path = - description = - exec = + description = + exec = Data providers ============== @@ -612,4 +612,4 @@ https://github.com/scrapy/slybot Brings together scrapy + scrapely to provide json-specs for spiders/items/etc Might be worth at least adopting spiders specs...? - Has a neat slybot/validation/schemas.json which validates the schematic + Has a neat slybot/validation/schemas.json which validates the schematic diff --git a/docs/source/_extras/readme.txt b/docs/source/_extras/readme.txt index 3285d0b375..19d6d22a67 100644 --- a/docs/source/_extras/readme.txt +++ b/docs/source/_extras/readme.txt @@ -1 +1 @@ -Directory used by sphinx \ No newline at end of file +Directory used by sphinx diff --git a/docs/source/basics.rst b/docs/source/basics.rst index 0d627b7691..9516ff847f 100644 --- a/docs/source/basics.rst +++ b/docs/source/basics.rst @@ -124,7 +124,7 @@ https://datasets.datalad.org provides a super-dataset consisting of datasets from various portals and sites. Many of them were crawled, and periodically updated, using `datalad-crawler `__ extension. The argument ``///`` can be used -as a shortcut that points to the superdataset located at https://datasets.datalad.org/. +as a shortcut that points to the superdataset located at https://datasets.datalad.org/. Here are three common examples in command line notation: ``datalad install ///`` @@ -132,7 +132,7 @@ Here are three common examples in command line notation: `datasets.datalad.org/` subdirectory under the current directory ``datalad install -r ///openfmri`` installs the openfmri superdataset into an `openfmri/` subdirectory. - Additionally, the ``-r`` flag recursively downloads all metadata of datasets + Additionally, the ``-r`` flag recursively downloads all metadata of datasets available from http://openfmri.org as subdatasets into the `openfmri/` subdirectory ``datalad install -g -J3 -r ///labs/haxby`` installs the superdataset of datasets released by the lab of Dr. James V. Haxby diff --git a/docs/source/basics_cmdline.rst.in b/docs/source/basics_cmdline.rst.in index 4e2b2efd84..f8261326c8 100644 --- a/docs/source/basics_cmdline.rst.in +++ b/docs/source/basics_cmdline.rst.in @@ -22,7 +22,7 @@ of basic options, and a list of available sub-commands. [--on-failure {ignore,continue,stop}] [--cmd] {create,install,get,publish,uninstall,drop,remove,update,create-sibling,create-sibling-github,unlock,save,search,metadata,aggregate-metadata,test,ls,clean,add-archive-content,download-url,run,rerun,addurls,export-archive,extract-metadata,export-to-figshare,no-annex,wtf,add-readme,annotate-paths,clone,create-test-dataset,diff,siblings,sshrun,subdatasets} ... - [ERROR ] Please specify the command + [ERROR ] Please specify the command ~ % # More comprehensive information is available via the --help @@ -32,13 +32,13 @@ long-option (we will truncate the output here) ~ % datalad --help | head -n20 Usage: datalad [global-opts] command [command-opts] - + DataLad provides a unified data distribution with the convenience of git-annex repositories as a backend. DataLad command line tools allow to manipulate (obtain, create, update, publish, etc.) datasets and their collections. - + *Commands for dataset operations* - + create Create a new dataset from scratch install @@ -65,9 +65,9 @@ again truncated) [--git-opts STRING] [--annex-opts STRING] [--annex-init-opts STRING] [--text-no-annex] [PATH] - + Create a new dataset from scratch. - + This command initializes a new dataset at a given location, or the current directory. The new dataset can optionally be registered in an existing superdataset (the new dataset's path needs to be located @@ -77,7 +77,7 @@ again truncated) laptop". This helps humans to identify data locations in distributed scenarios. By default an identifier comprised of user and machine name, plus path will be generated. - + ______________________________________ diff --git a/docs/source/basics_nesteddatasets.rst.in b/docs/source/basics_nesteddatasets.rst.in index a7b2d76027..50bd01aca0 100644 --- a/docs/source/basics_nesteddatasets.rst.in +++ b/docs/source/basics_nesteddatasets.rst.in @@ -12,7 +12,7 @@ Let's create a dataset .. code-block:: ansi-color ~ % datalad create demo - [INFO ] Creating a new annex repo at /demo/demo + [INFO ] Creating a new annex repo at /demo/demo create(ok): /demo/demo (dataset) ~ % cd demo @@ -32,7 +32,7 @@ new dataset in a parent dataset .. code-block:: ansi-color ~/demo % datalad create -d . sub1 - [INFO ] Creating a new annex repo at /demo/demo/sub1 + [INFO ] Creating a new annex repo at /demo/demo/sub1 add(ok): sub1 (dataset) [added new subdataset] add(notneeded): sub1 (dataset) [nothing to add from /demo/demo/sub1] add(notneeded): .gitmodules (file) [already included in the dataset] @@ -57,7 +57,7 @@ Of course subdatasets can be nested .. code-block:: ansi-color ~/demo % datalad create -d . sub1/justadir/sub2 - [INFO ] Creating a new annex repo at /demo/demo/sub1/justadir/sub2 + [INFO ] Creating a new annex repo at /demo/demo/sub1/justadir/sub2 add(ok): sub1/justadir/sub2 (dataset) [added new subdataset] add(notneeded): sub1/justadir/sub2 (dataset) [nothing to add from /demo/demo/sub1/justadir/sub2] add(notneeded): sub1/.gitmodules (file) [already included in the dataset] @@ -101,9 +101,9 @@ subdataset was modified (use "git add ..." to update what will be committed) (use "git checkout -- ..." to discard changes in working directory) (commit or discard the untracked or modified content in submodules) - + modified: sub1 (untracked content) - + no changes added to commit (use "git add" and/or "git commit -a") diff --git a/docs/source/design/cli.rst b/docs/source/design/cli.rst index 6f5ce023aa..95cd1fefb4 100644 --- a/docs/source/design/cli.rst +++ b/docs/source/design/cli.rst @@ -130,4 +130,3 @@ processing of result recorded yielded by a command. If an underlying command returns a Python generator it is unwound to trigger the respective underlying processing. The handler performs no error handling. This is left to the main command line entry point. - diff --git a/docs/source/design/credentials.rst b/docs/source/design/credentials.rst index 58ea561fa5..e388e7ba9b 100644 --- a/docs/source/design/credentials.rst +++ b/docs/source/design/credentials.rst @@ -11,7 +11,7 @@ Credential management This specification describes the current implementation. -Various components of DataLad need to be passed credentials to interact with services that require authentication. +Various components of DataLad need to be passed credentials to interact with services that require authentication. This includes downloading files, but also things like REST API usage or authenticated cloning. Key components of DataLad's credential management are credentials types, providers, authenticators and downloaders. diff --git a/docs/source/design/drop.rst b/docs/source/design/drop.rst index f6f6e2d460..6b6b14e502 100644 --- a/docs/source/design/drop.rst +++ b/docs/source/design/drop.rst @@ -44,7 +44,7 @@ an annex as ``dead`` on removal of a repository (`#3887 `__). §6 Like :command:`get`, drop supports parallelization `#1953 -`__ +`__ §7 `datalad drop` is not intended to be a comprehensive frontend to `git annex drop` (e.g. limited support for e.g. `#1482 diff --git a/docs/source/design/pos_vs_kw_parameters.rst b/docs/source/design/pos_vs_kw_parameters.rst index 9ba1161e4f..7498c9ab90 100644 --- a/docs/source/design/pos_vs_kw_parameters.rst +++ b/docs/source/design/pos_vs_kw_parameters.rst @@ -48,4 +48,4 @@ Regular functions and methods Use of ``*`` is encouraged for any function (or method) with keyword arguments. Generally, ``*`` should come before the first keyword argument, but similarly to the Interfaces above, it is left to the discretion of the developer to possibly allocate some (just few) arguments which could be used positionally if -specified. \ No newline at end of file +specified. diff --git a/docs/source/design/progress_reporting.rst b/docs/source/design/progress_reporting.rst index ce16bd86d4..6505a24211 100644 --- a/docs/source/design/progress_reporting.rst +++ b/docs/source/design/progress_reporting.rst @@ -156,7 +156,7 @@ level than intermediate updates. It is also possible to single out particular intermediate events, and report them at a higher level. If no `noninteractive_level` is specified, the progress update is unconditionally -logged at the level implied by the given logger callable. +logged at the level implied by the given logger callable. Reporting progress with `with_(result_)progress()` diff --git a/docs/source/design/python_imports.rst b/docs/source/design/python_imports.rst index 02177401d3..941346c3dc 100644 --- a/docs/source/design/python_imports.rst +++ b/docs/source/design/python_imports.rst @@ -41,9 +41,9 @@ The following rules apply to any ``import`` statement in the code base: - DataLad core (absolute imports) - DataLad extensions - + - DataLad core ("local" relative imports) - + Sorting imports can be aided by https://github.com/PyCQA/isort (e.g. ``python -m isort -m3 --fgw 2 --tc ``). @@ -63,15 +63,14 @@ Examples ensure_unicode, get_dataset_root as gdr, ) - + In the `datalad/submodule/tests/test_mod.py` test file demonstrating an "exception" to absolute imports - rule where test files are accompanying corresponding files of the underlying module:: - + rule where test files are accompanying corresponding files of the underlying module:: + import os - + from datalad.utils import ensure_list - + from ..mod import func1 from datalad.tests.utils_pytest import assert_true - diff --git a/docs/source/design/standard_parameters.rst b/docs/source/design/standard_parameters.rst index 52b0ee80ac..e5c2f2badd 100644 --- a/docs/source/design/standard_parameters.rst +++ b/docs/source/design/standard_parameters.rst @@ -45,4 +45,4 @@ The parameters are listed in their command-line form, but similar names and desc Certain standard parameters will have their own design document. -Please refer to those documents for more in-depth information. \ No newline at end of file +Please refer to those documents for more in-depth information. diff --git a/docs/source/design/user_messaging.rst b/docs/source/design/user_messaging.rst index eda104eaa6..2fbc1c9543 100644 --- a/docs/source/design/user_messaging.rst +++ b/docs/source/design/user_messaging.rst @@ -122,4 +122,4 @@ messaging methods: .. _Result yielding: https://github.com/datalad/datalad/blob/a8d7c63b763aacfbca15925bb1562a62b4448ea6/datalad/core/local/status.py#L402-L426 .. _Exception handling: https://github.com/datalad/datalad/blob/a8d7c63b763aacfbca15925bb1562a62b4448ea6/datalad/core/local/status.py#L149-L150 .. _Logging: https://github.com/datalad/datalad/blob/a8d7c63b763aacfbca15925bb1562a62b4448ea6/datalad/core/local/status.py#L158 -.. _UI messaging: https://github.com/datalad/datalad/blob/a8d7c63b763aacfbca15925bb1562a62b4448ea6/datalad/core/local/status.py#L438-L457 \ No newline at end of file +.. _UI messaging: https://github.com/datalad/datalad/blob/a8d7c63b763aacfbca15925bb1562a62b4448ea6/datalad/core/local/status.py#L438-L457 diff --git a/docs/source/glossary.rst b/docs/source/glossary.rst index fa65fa11b1..f89793aa56 100644 --- a/docs/source/glossary.rst +++ b/docs/source/glossary.rst @@ -59,4 +59,4 @@ them to the corresponding Git_/git-annex_ concepts. .. _Git: https://git-scm.com .. _Git-annex: http://git-annex.branchable.com .. _`Command Line Interface`: https://en.wikipedia.org/wiki/Command-line_interface -.. _shell: https://en.wikipedia.org/wiki/Shell_(computing) \ No newline at end of file +.. _shell: https://en.wikipedia.org/wiki/Shell_(computing) diff --git a/docs/utils/pygments_ansi_color.py b/docs/utils/pygments_ansi_color.py index 205f4dedcd..79776d3915 100644 --- a/docs/utils/pygments_ansi_color.py +++ b/docs/utils/pygments_ansi_color.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- """Pygments lexer for text containing ANSI color codes.""" -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import ( + absolute_import, + unicode_literals, +) import itertools import re @@ -9,7 +11,6 @@ import pygments.lexer import pygments.token - Color = pygments.token.Token.Color _ansi_code_to_color = { From 7e7ad485ce7058a883f78fb757e8596104729621 Mon Sep 17 00:00:00 2001 From: bpinsard Date: Fri, 30 Aug 2024 10:37:11 -0400 Subject: [PATCH 2/2] fix comment --- datalad/core/distributed/clone.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/datalad/core/distributed/clone.py b/datalad/core/distributed/clone.py index 7576299f68..6a1b4e5a2f 100644 --- a/datalad/core/distributed/clone.py +++ b/datalad/core/distributed/clone.py @@ -50,7 +50,7 @@ rmtree, ) -from .clone_utils import ( # needed because other code imports it from here; RIA imports needed b/c datalad-next imports it from here ATM;; Remove after core was released and next dropped the ria patch. +from .clone_utils import ( # needed because other code imports it from here _check_autoenable_special_remotes, _format_clone_errors, _generate_candidate_clone_sources, @@ -60,6 +60,11 @@ _test_existing_clone_target, _try_clone_candidates, decode_source_spec, +) + +from .clone_utils import ( # isort: skip + # RIA imports needed b/c datalad-next imports it from here ATM + # Remove after core was released and next dropped the ria patch. postclone_preannex_cfg_ria, postclonecfg_ria, )