From 5e994939941ddffa3f1d588de9ce18ba4d8653c6 Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Thu, 6 Sep 2018 16:18:42 -0400 Subject: [PATCH 01/12] Add initial GWArgSpec class with tests --- girder_worker_utils/decorators.py | 160 +++++++++++++++++- girder_worker_utils/tests/conftest.py | 5 + girder_worker_utils/tests/decorators_test.py | 137 +++++++++++++++ .../tests/py3_decorators_test.py | 112 ++++++++++++ 4 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 girder_worker_utils/tests/conftest.py create mode 100644 girder_worker_utils/tests/py3_decorators_test.py diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index ca21847..186e81a 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -1,8 +1,9 @@ from inspect import getdoc try: - from inspect import signature + from inspect import signature, Parameter except ImportError: # pragma: nocover - from funcsigs import signature + from funcsigs import signature, Parameter + import six @@ -23,6 +24,161 @@ def get_description_attribute(func): return description +class Argument(object): + def __init__(self, name, **kwargs): + self.name = name + for k, v in six.iteritems(kwargs): + setattr(self, k, v) + + +class KWArg(Argument): + def __init__(self, name, **kwargs): + super(KWArg, self).__init__(name, **kwargs) + + +class PosArg(Argument): + def __init__(self, name, **kwargs): + super(PosArg, self).__init__(name, **kwargs) + + +class Varargs(Argument): + def __init__(self, name, **kwargs): + super(Varargs, self).__init__(name, **kwargs) + + +class Kwargs(Argument): + def __init__(self, name, **kwargs): + super(Kwargs, self).__init__(name, **kwargs) + + +# class Return(Argument): +# def __init__(self, name, **kwargs): +# self.name = name +# for k, v in six.iteritems(kwargs): +# setattr(self, k, v) + + +class GWArgSpec(object): + _parameter_repr = ['POSITIONAL_ONLY', + 'POSITIONAL_OR_KEYWORD', + 'VAR_POSITIONAL', + 'KEYWORD_ONLY', + 'VAR_KEYWORD'] + + def __init__(self, func): + self._metadata = {} + self._signature = signature(func) + + def __repr__(self): + # TODO - make less ugly + return "<{}(".format(self.__class__.__name__) + ", ".join(["{}:{}".format( + name, self._parameter_repr[self._signature.parameters[name].kind]) + for name in self._signature.parameters]) + ")>" + + def __getitem__(self, key): + return self._construct_argument( + self._get_class(self._signature.parameters[key]), key) + + def _construct_argument(self, cls, name, **kwargs): + p = self._signature.parameters[name] + metadata = {} + + if p.default != p.empty: + metadata['default'] = p.default + if p.annotation != p.empty: + # TODO: make sure annotation is a type and not just garbage + metadata['data_type'] = p.annotation + + metadata.update(self._metadata.get(name, {})) + + return cls(name, **metadata) + + def _is_varargs(self, p): + return p.kind == Parameter.VAR_POSITIONAL + + def _is_kwargs(self, p): + return p.kind == Parameter.VAR_KEYWORD + + def _is_kwarg(self, p): + return p.kind == Parameter.KEYWORD_ONLY or ( + p.kind == Parameter.POSITIONAL_OR_KEYWORD and p.default != p.empty) + + def _is_posarg(self, p): + return p.kind == Parameter.POSITIONAL_ONLY or ( + p.kind == Parameter.POSITIONAL_OR_KEYWORD and p.default == p.empty) + + def _get_class(self, p): + if self._is_varargs(p): + return Varargs + elif self._is_kwargs(p): + return Kwargs + elif self._is_posarg(p): + return PosArg + elif self._is_kwarg(p): + return KWArg + else: + raise RuntimeError("Could not determine parameter type!") + + def set_metadata(self, name, key, value): + if name not in self._signature.parameters: + raise RuntimeError("{} is not a valid argument to this function!") + + if name not in self._metadata: + self._metadata[name] = {} + + self._metadata[name][key] = value + + + @property + def arguments(self): + return [ + self._construct_argument( + self._get_class(self._signature.parameters[name]), name) + for name in self._signature.parameters] + + @property + def varargs(self): + for name in self._signature.parameters: + if self._is_varargs(self._signature.parameters[name]): + return self._construct_argument(Varargs, name) + return None + + @property + def kwargs(self): + for name in self._signature.parameters: + if self._is_kwargs(self._signature.parameters[name]): + return self._construct_argument(Kwargs, name) + return None + + @property + def positional_args(self): + return [arg for arg in self.arguments if isinstance(arg, PosArg)] + + @property + def keyword_args(self): + return [arg for arg in self.arguments if isinstance(arg, KWArg)] + + +def parameter(name, **kwargs): + if not isinstance(name, six.string_types): + raise TypeError('Expected argument name to be a string') + + data_type = kwargs.get("data_type", None) + if data_type is not None and callable(data_type): + kwargs['data_type'] = data_type(name, **kwargs) + + def argument_wrapper(func): + if not hasattr(func, "_girder_spec"): + func._girder_spec = GWArgSpec(func) + + for key, value in six.iteritems(kwargs): + func._girder_spec.set_metadata(name, key, value) + + return func + + return argument_wrapper + + def argument(name, data_type, *args, **kwargs): """Describe an argument to a function as a function decorator. diff --git a/girder_worker_utils/tests/conftest.py b/girder_worker_utils/tests/conftest.py new file mode 100644 index 0000000..98fd7a6 --- /dev/null +++ b/girder_worker_utils/tests/conftest.py @@ -0,0 +1,5 @@ +import six + +collect_ignore = [] +if six.PY2: + collect_ignore.append("py3_decorators_test.py") diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index 85a9c78..cf262cf 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -172,3 +172,140 @@ def test_unhandled_input_binding(): arg = argument('arg', types.Integer) with pytest.raises(ValueError): decorators.get_input_data(arg, {}) + + + +########################### + + +import six + +from girder_worker_utils.decorators import parameter +from girder_worker_utils.decorators import ( + GWArgSpec, + Varargs, + Kwargs, + PosArg, + KWArg) + + +def arg(a): pass # noqa +def varargs(*args): pass # noqa +def kwarg(a='test'): pass # noqa +def kwargs(**kwargs): pass # noqa +def arg_arg(a, b): pass # noqa +def arg_varargs(a, *args): pass # noqa +def arg_kwarg(a, b='test'): pass # noqa +def arg_kwargs(a, **kwargs): pass # noqa +def kwarg_varargs(a='test', *args): pass # noqa +def kwarg_kwarg(a='testa', b='testb'): pass # noqa +def kwarg_kwargs(a='test', **kwargs): pass # noqa +def arg_kwarg_varargs(a, b='test', *args): pass # noqa +def arg_kwarg_kwargs(a, b='test', **kwargs): pass # noqa +def arg_kwarg_varargs_kwargs(a, b='test', *args, **kwargs): pass # noqa + + + +@pytest.mark.parametrize('func,classes', [ + (arg, [PosArg]), + (varargs, [Varargs]), + (kwarg, [KWArg]), + (kwargs, [Kwargs]), + (arg_arg, [PosArg, PosArg]), + (arg_varargs, [PosArg, Varargs]), + (arg_kwarg, [PosArg, KWArg]), + (arg_kwargs, [PosArg, Kwargs]), + (kwarg_varargs, [KWArg, Varargs]), + (kwarg_kwarg, [KWArg, KWArg]), + (kwarg_kwargs, [KWArg, Kwargs]), + (arg_kwarg_varargs, [PosArg, KWArg, Varargs]), + (arg_kwarg_kwargs, [PosArg, KWArg, Kwargs]), + (arg_kwarg_varargs_kwargs, [PosArg, KWArg, Varargs, Kwargs]) +]) +def test_GWArgSpec_arguments_returns_expected_classes(func, classes): + spec = GWArgSpec(func) + assert len(spec.arguments) == len(classes) + for arg, cls in zip(spec.arguments, classes): + assert isinstance(arg, cls) + + +no_varargs = [arg, kwarg, kwargs, arg_arg, arg_kwarg, + arg_kwargs, kwarg_kwarg, kwarg_kwargs, + arg_kwarg_kwargs] + +@pytest.mark.parametrize('func', no_varargs) +def test_GWArgSpec_varargs_returns_None(func): + spec = GWArgSpec(func) + assert spec.varargs is None + + +with_varargs = [varargs, arg_varargs, kwarg_varargs, + arg_kwarg_varargs, arg_kwarg_varargs_kwargs] + +@pytest.mark.parametrize('func', with_varargs) +def test_GWArgSpec_varargs_returns_Vararg(func): + spec = GWArgSpec(func) + assert isinstance(spec.varargs, Varargs) + + +@pytest.mark.parametrize('func,names', [ + (arg, ["a"]), + (arg_arg, ["a", "b"]), + (arg_varargs, ["a"]), + (arg_kwarg, ["a"]), + (arg_kwargs, ["a"]), + (arg_kwarg_kwargs, ["a"]), + (arg_kwarg_varargs_kwargs, ["a"]) +]) +def test_GWArgSpec_positional_args_correct_names(func, names): + spec = GWArgSpec(func) + assert len(spec.positional_args) == len(names) + for p, n in zip(spec.positional_args, names): + assert isinstance(p, PosArg) + assert p.name == n + + +# TODO positional_args returns None test + +@pytest.mark.parametrize('func,names', [ + (kwarg, ['a']), + (arg_kwarg, ['b']), + (kwarg_varargs, ['a']), + (kwarg_kwarg, ['a', 'b']), + (kwarg_kwargs, ['a']), + (arg_kwarg_varargs, ['b']), + (arg_kwarg_kwargs, ['b']), + (arg_kwarg_varargs_kwargs, ['b']), +]) +def test_GWArgSpec_keyword_args_correct_names(func, names): + spec = GWArgSpec(func) + assert len(spec.keyword_args) == len(names) + for p, n in zip(spec.keyword_args, names): + assert isinstance(p, KWArg) + assert p.name == n + +# TODO keyword_args returns None test +@pytest.mark.parametrize('func,defaults', [ + (kwarg, ['test']), + (arg_kwarg, ['test']), + (kwarg_varargs, ['test']), + (kwarg_kwarg, ['testa', 'testb']), + (kwarg_kwargs, ['test']), + (arg_kwarg_varargs, ['test']), + (arg_kwarg_kwargs, ['test']), + (arg_kwarg_varargs_kwargs, ['test']), +]) +def test_GWArgSpec_keyword_args_have_defaults(func, defaults): + spec = GWArgSpec(func) + assert len(spec.keyword_args) == len(defaults) + for p, d in zip(spec.keyword_args, defaults): + assert hasattr(p, 'default') + assert p.default == d + + +def test_parameter_decorator_adds_metadata(): + @parameter('a', test='TEST') + def arg(a): pass # noqa + + assert hasattr(arg._girder_spec['a'], 'test') + assert arg._girder_spec['a'].test == 'TEST' diff --git a/girder_worker_utils/tests/py3_decorators_test.py b/girder_worker_utils/tests/py3_decorators_test.py new file mode 100644 index 0000000..cf98436 --- /dev/null +++ b/girder_worker_utils/tests/py3_decorators_test.py @@ -0,0 +1,112 @@ +import pytest +import six +from girder_worker_utils.decorators import ( + GWArgSpec, + Varargs, + Kwargs, + PosArg, + KWArg) + +def arg_varargs_kwarg(a, *args, b='test'): pass +def arg_varargs_kwarg_no_default(a, *args, b): pass +def arg_emptyvarargs_kwarg(a, *, b='test'): pass +def arg_emptyvarargs_kwarg_no_default(a, *, b): pass + +def arg_with_annotation(a: int): pass + + +@pytest.mark.parametrize('func,classes', [ + (arg_varargs_kwarg, [PosArg, Varargs, KWArg]), + (arg_varargs_kwarg_no_default, [PosArg, Varargs, KWArg]), + (arg_emptyvarargs_kwarg, [PosArg, KWArg]), + (arg_emptyvarargs_kwarg_no_default, [PosArg, KWArg]) +]) +def test_GWArgSpec_arguments_returns_expected_classes(func, classes): + spec = GWArgSpec(func) + assert len(spec.arguments) == len(classes) + for arg, cls in zip(spec.arguments, classes): + assert isinstance(arg, cls) + + +no_varargs = [arg_emptyvarargs_kwarg, + arg_emptyvarargs_kwarg_no_default] + + +@pytest.mark.parametrize('func', no_varargs) +def test_GWArgSpec_varargs_returns_None(func): + spec = GWArgSpec(func) + assert spec.varargs is None + + +with_varargs = [ + arg_varargs_kwarg, + arg_varargs_kwarg_no_default] + + +@pytest.mark.parametrize('func', with_varargs) +def test_GWArgSpec_varargs_returns_Vararg(func): + spec = GWArgSpec(func) + assert isinstance(spec.varargs, Varargs) + + +@pytest.mark.parametrize('func,names', [ + (arg_varargs_kwarg, ["a"]), + (arg_varargs_kwarg_no_default, ["a"]), + (arg_emptyvarargs_kwarg, ["a"]), + (arg_emptyvarargs_kwarg_no_default, ["a"]), +]) +def test_GWArgSpec_positional_args_correct_names(func, names): + spec = GWArgSpec(func) + assert len(spec.positional_args) == len(names) + for p, n in zip(spec.positional_args, names): + assert isinstance(p, PosArg) + assert p.name == n + + +# TODO positional_args returns None test + +@pytest.mark.parametrize('func,names', [ + (arg_varargs_kwarg, ['b']), + (arg_varargs_kwarg_no_default, ['b']), + (arg_emptyvarargs_kwarg, ['b']), + (arg_emptyvarargs_kwarg_no_default, ['b']) +]) +def test_GWArgSpec_keyword_args_correct_names(func, names): + spec = GWArgSpec(func) + assert len(spec.keyword_args) == len(names) + for p, n in zip(spec.keyword_args, names): + assert isinstance(p, KWArg) + assert p.name == n + +# TODO keyword_args returns None test + +@pytest.mark.parametrize('func,defaults', [ + (arg_varargs_kwarg, ['test']), + (arg_emptyvarargs_kwarg, ['test']) +]) +def test_GWArgSpec_keyword_args_have_defaults(func, defaults): + spec = GWArgSpec(func) + assert len(spec.keyword_args) == len(defaults) + for p, d in zip(spec.keyword_args, defaults): + assert hasattr(p, 'default') + assert p.default == d + +@pytest.mark.parametrize('func', [ + arg_varargs_kwarg_no_default, + arg_emptyvarargs_kwarg_no_default +]) +def test_GWArgSpec_keyword_args_with_no_defaults_have_no_defaults(func): + spec = GWArgSpec(func) + for p in spec.keyword_args: + assert not hasattr(p, "default") + + +@pytest.mark.parametrize('func,data_types', [ + (arg_with_annotation, [int]) +]) +def test_GWArgSpec_positional_args_with_annotation_have_data_type(func, data_types): + spec = GWArgSpec(func) + assert len(spec.positional_args) == len(data_types) + for p, d in zip(spec.positional_args, data_types): + assert hasattr(p, "data_type") + assert p.data_type == d From 3bbbab08983e37fa8599e6b15029a08cdf60327b Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Mon, 10 Sep 2018 12:37:53 -0400 Subject: [PATCH 02/12] Decorate celery task directly treat as regular callable. --- girder_worker_utils/decorators.py | 104 +++++++++++------- girder_worker_utils/tests/decorators_test.py | 49 +++++---- .../tests/py3_decorators_test.py | 47 ++++---- 3 files changed, 112 insertions(+), 88 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index 186e81a..e8248b4 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -1,4 +1,4 @@ -from inspect import getdoc +from inspect import getdoc, cleandoc try: from inspect import signature, Parameter except ImportError: # pragma: nocover @@ -17,8 +17,8 @@ class MissingInputException(Exception): def get_description_attribute(func): """Get the private description attribute from a function.""" - func = getattr(func, 'run', func) - description = getattr(func, '_girder_description', None) + # func = getattr(func, 'run', func) + description = getattr(func, GWFuncDesc._func_desc_attr, None) if description is None: raise MissingDescriptionException('Function is missing description decorators') return description @@ -30,42 +30,44 @@ def __init__(self, name, **kwargs): for k, v in six.iteritems(kwargs): setattr(self, k, v) - -class KWArg(Argument): - def __init__(self, name, **kwargs): - super(KWArg, self).__init__(name, **kwargs) - - -class PosArg(Argument): - def __init__(self, name, **kwargs): - super(PosArg, self).__init__(name, **kwargs) - - -class Varargs(Argument): - def __init__(self, name, **kwargs): - super(Varargs, self).__init__(name, **kwargs) - - -class Kwargs(Argument): - def __init__(self, name, **kwargs): - super(Kwargs, self).__init__(name, **kwargs) +# No default value for this argument +class Arg(Argument): pass +# Has a default argument for the value +class KWArg(Argument): pass +class Varargs(Argument): pass +class Kwargs(Argument): pass +# class Return(Argument): pass -# class Return(Argument): -# def __init__(self, name, **kwargs): -# self.name = name -# for k, v in six.iteritems(kwargs): -# setattr(self, k, v) +def _clean_function_doc(f): + doc = getdoc(f) or '' + if isinstance(doc, bytes): + doc = doc.decode('utf-8') + else: + doc = cleandoc(doc) + return doc -class GWArgSpec(object): +class GWFuncDesc(object): + _func_desc_attr = "_gw_function_description" _parameter_repr = ['POSITIONAL_ONLY', 'POSITIONAL_OR_KEYWORD', 'VAR_POSITIONAL', 'KEYWORD_ONLY', 'VAR_KEYWORD'] + @classmethod + def get_description(cls, func): + # HACK - potentially unwrap celery task + # func = getattr(func, 'run', func) + if hasattr(func, cls._func_desc_attr) and \ + isinstance(getattr(func, cls._func_desc_attr), cls): + return getattr(func, cls._func_desc_attr) + return None + def __init__(self, func): + self.func_name = func.__name__ + self.func_help = _clean_function_doc(func) self._metadata = {} self._signature = signature(func) @@ -113,12 +115,13 @@ def _get_class(self, p): elif self._is_kwargs(p): return Kwargs elif self._is_posarg(p): - return PosArg + return Arg elif self._is_kwarg(p): return KWArg else: raise RuntimeError("Could not determine parameter type!") + def set_metadata(self, name, key, value): if name not in self._signature.parameters: raise RuntimeError("{} is not a valid argument to this function!") @@ -128,31 +131,36 @@ def set_metadata(self, name, key, value): self._metadata[name][key] = value - @property def arguments(self): + # Only return arguments if we've declared them as paramaters + # This prevents us from returning things like 'self' of bound + # methods (e.g. celery tasks) etc. This is a dubious design + # decision. return [ self._construct_argument( self._get_class(self._signature.parameters[name]), name) - for name in self._signature.parameters] + for name in self._signature.parameters if name in self._metadata] @property def varargs(self): for name in self._signature.parameters: - if self._is_varargs(self._signature.parameters[name]): + if name in self._metadata and \ + self._is_varargs(self._signature.parameters[name]): return self._construct_argument(Varargs, name) return None @property def kwargs(self): for name in self._signature.parameters: - if self._is_kwargs(self._signature.parameters[name]): + if name in self._metadata and \ + self._is_kwargs(self._signature.parameters[name]): return self._construct_argument(Kwargs, name) return None @property def positional_args(self): - return [arg for arg in self.arguments if isinstance(arg, PosArg)] + return [arg for arg in self.arguments if isinstance(arg, Arg)] @property def keyword_args(self): @@ -168,11 +176,25 @@ def parameter(name, **kwargs): kwargs['data_type'] = data_type(name, **kwargs) def argument_wrapper(func): - if not hasattr(func, "_girder_spec"): - func._girder_spec = GWArgSpec(func) + if not hasattr(func, GWFuncDesc._func_desc_attr): + setattr(func, GWFuncDesc._func_desc_attr, GWFuncDesc(func)) - for key, value in six.iteritems(kwargs): - func._girder_spec.set_metadata(name, key, value) + desc = getattr(func, GWFuncDesc._func_desc_attr) + + # Make sure the metadata key exists even if we don't set any + # values on it. This ensures that metadata's keys represent + # the full list of parameters that have been identified by the + # user (even if there is no actual metadata associated with + # the argument). + desc._metadata[name] = {} + + for key, value in six.iteritems(kwargs): + desc.set_metadata(name, key, value) + + def description(): + return getattr(func, GWFuncDesc._func_desc_attr) + + func.description = description return func @@ -194,8 +216,10 @@ def argument(name, data_type, *args, **kwargs): data_type = data_type(name, *args, **kwargs) def argument_wrapper(func): - func._girder_description = getattr(func, '_girder_description', {}) - args = func._girder_description.setdefault('arguments', []) + setattr(func, GWFuncDesc._func_desc_attr, + getattr(func, GWFuncDesc._func_desc_attr, {})) + + args = getattr(func, GWFuncDesc._func_desc_attr).setdefault('arguments', []) sig = signature(func) if name not in sig.parameters: diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index cf262cf..d9cc530 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -182,10 +182,10 @@ def test_unhandled_input_binding(): from girder_worker_utils.decorators import parameter from girder_worker_utils.decorators import ( - GWArgSpec, + GWFuncDesc, Varargs, Kwargs, - PosArg, + Arg, KWArg) @@ -207,23 +207,23 @@ def arg_kwarg_varargs_kwargs(a, b='test', *args, **kwargs): pass # noqa @pytest.mark.parametrize('func,classes', [ - (arg, [PosArg]), + (arg, [Arg]), (varargs, [Varargs]), (kwarg, [KWArg]), (kwargs, [Kwargs]), - (arg_arg, [PosArg, PosArg]), - (arg_varargs, [PosArg, Varargs]), - (arg_kwarg, [PosArg, KWArg]), - (arg_kwargs, [PosArg, Kwargs]), + (arg_arg, [Arg, Arg]), + (arg_varargs, [Arg, Varargs]), + (arg_kwarg, [Arg, KWArg]), + (arg_kwargs, [Arg, Kwargs]), (kwarg_varargs, [KWArg, Varargs]), (kwarg_kwarg, [KWArg, KWArg]), (kwarg_kwargs, [KWArg, Kwargs]), - (arg_kwarg_varargs, [PosArg, KWArg, Varargs]), - (arg_kwarg_kwargs, [PosArg, KWArg, Kwargs]), - (arg_kwarg_varargs_kwargs, [PosArg, KWArg, Varargs, Kwargs]) + (arg_kwarg_varargs, [Arg, KWArg, Varargs]), + (arg_kwarg_kwargs, [Arg, KWArg, Kwargs]), + (arg_kwarg_varargs_kwargs, [Arg, KWArg, Varargs, Kwargs]) ]) -def test_GWArgSpec_arguments_returns_expected_classes(func, classes): - spec = GWArgSpec(func) +def test_GWFuncDesc_arguments_returns_expected_classes(func, classes): + spec = GWFuncDesc(func) assert len(spec.arguments) == len(classes) for arg, cls in zip(spec.arguments, classes): assert isinstance(arg, cls) @@ -234,8 +234,8 @@ def test_GWArgSpec_arguments_returns_expected_classes(func, classes): arg_kwarg_kwargs] @pytest.mark.parametrize('func', no_varargs) -def test_GWArgSpec_varargs_returns_None(func): - spec = GWArgSpec(func) +def test_GWFuncDesc_varargs_returns_None(func): + spec = GWFuncDesc(func) assert spec.varargs is None @@ -243,8 +243,8 @@ def test_GWArgSpec_varargs_returns_None(func): arg_kwarg_varargs, arg_kwarg_varargs_kwargs] @pytest.mark.parametrize('func', with_varargs) -def test_GWArgSpec_varargs_returns_Vararg(func): - spec = GWArgSpec(func) +def test_GWFuncDesc_varargs_returns_Vararg(func): + spec = GWFuncDesc(func) assert isinstance(spec.varargs, Varargs) @@ -257,11 +257,11 @@ def test_GWArgSpec_varargs_returns_Vararg(func): (arg_kwarg_kwargs, ["a"]), (arg_kwarg_varargs_kwargs, ["a"]) ]) -def test_GWArgSpec_positional_args_correct_names(func, names): - spec = GWArgSpec(func) +def test_GWFuncDesc_positional_args_correct_names(func, names): + spec = GWFuncDesc(func) assert len(spec.positional_args) == len(names) for p, n in zip(spec.positional_args, names): - assert isinstance(p, PosArg) + assert isinstance(p, Arg) assert p.name == n @@ -277,8 +277,8 @@ def test_GWArgSpec_positional_args_correct_names(func, names): (arg_kwarg_kwargs, ['b']), (arg_kwarg_varargs_kwargs, ['b']), ]) -def test_GWArgSpec_keyword_args_correct_names(func, names): - spec = GWArgSpec(func) +def test_GWFuncDesc_keyword_args_correct_names(func, names): + spec = GWFuncDesc(func) assert len(spec.keyword_args) == len(names) for p, n in zip(spec.keyword_args, names): assert isinstance(p, KWArg) @@ -295,8 +295,8 @@ def test_GWArgSpec_keyword_args_correct_names(func, names): (arg_kwarg_kwargs, ['test']), (arg_kwarg_varargs_kwargs, ['test']), ]) -def test_GWArgSpec_keyword_args_have_defaults(func, defaults): - spec = GWArgSpec(func) +def test_GWFuncDesc_keyword_args_have_defaults(func, defaults): + spec = GWFuncDesc(func) assert len(spec.keyword_args) == len(defaults) for p, d in zip(spec.keyword_args, defaults): assert hasattr(p, 'default') @@ -305,7 +305,8 @@ def test_GWArgSpec_keyword_args_have_defaults(func, defaults): def test_parameter_decorator_adds_metadata(): @parameter('a', test='TEST') - def arg(a): pass # noqa + def arg(a): + pass assert hasattr(arg._girder_spec['a'], 'test') assert arg._girder_spec['a'].test == 'TEST' diff --git a/girder_worker_utils/tests/py3_decorators_test.py b/girder_worker_utils/tests/py3_decorators_test.py index cf98436..fc614f4 100644 --- a/girder_worker_utils/tests/py3_decorators_test.py +++ b/girder_worker_utils/tests/py3_decorators_test.py @@ -1,28 +1,27 @@ import pytest import six from girder_worker_utils.decorators import ( - GWArgSpec, + GWFuncDesc, Varargs, Kwargs, - PosArg, + Arg, KWArg) def arg_varargs_kwarg(a, *args, b='test'): pass def arg_varargs_kwarg_no_default(a, *args, b): pass def arg_emptyvarargs_kwarg(a, *, b='test'): pass def arg_emptyvarargs_kwarg_no_default(a, *, b): pass - def arg_with_annotation(a: int): pass @pytest.mark.parametrize('func,classes', [ - (arg_varargs_kwarg, [PosArg, Varargs, KWArg]), - (arg_varargs_kwarg_no_default, [PosArg, Varargs, KWArg]), - (arg_emptyvarargs_kwarg, [PosArg, KWArg]), - (arg_emptyvarargs_kwarg_no_default, [PosArg, KWArg]) + (arg_varargs_kwarg, [Arg, Varargs, KWArg]), + (arg_varargs_kwarg_no_default, [Arg, Varargs, KWArg]), + (arg_emptyvarargs_kwarg, [Arg, KWArg]), + (arg_emptyvarargs_kwarg_no_default, [Arg, KWArg]) ]) -def test_GWArgSpec_arguments_returns_expected_classes(func, classes): - spec = GWArgSpec(func) +def test_GWFuncDesc_arguments_returns_expected_classes(func, classes): + spec = GWFuncDesc(func) assert len(spec.arguments) == len(classes) for arg, cls in zip(spec.arguments, classes): assert isinstance(arg, cls) @@ -33,8 +32,8 @@ def test_GWArgSpec_arguments_returns_expected_classes(func, classes): @pytest.mark.parametrize('func', no_varargs) -def test_GWArgSpec_varargs_returns_None(func): - spec = GWArgSpec(func) +def test_GWFuncDesc_varargs_returns_None(func): + spec = GWFuncDesc(func) assert spec.varargs is None @@ -44,8 +43,8 @@ def test_GWArgSpec_varargs_returns_None(func): @pytest.mark.parametrize('func', with_varargs) -def test_GWArgSpec_varargs_returns_Vararg(func): - spec = GWArgSpec(func) +def test_GWFuncDesc_varargs_returns_Vararg(func): + spec = GWFuncDesc(func) assert isinstance(spec.varargs, Varargs) @@ -55,11 +54,11 @@ def test_GWArgSpec_varargs_returns_Vararg(func): (arg_emptyvarargs_kwarg, ["a"]), (arg_emptyvarargs_kwarg_no_default, ["a"]), ]) -def test_GWArgSpec_positional_args_correct_names(func, names): - spec = GWArgSpec(func) +def test_GWFuncDesc_positional_args_correct_names(func, names): + spec = GWFuncDesc(func) assert len(spec.positional_args) == len(names) for p, n in zip(spec.positional_args, names): - assert isinstance(p, PosArg) + assert isinstance(p, Arg) assert p.name == n @@ -71,8 +70,8 @@ def test_GWArgSpec_positional_args_correct_names(func, names): (arg_emptyvarargs_kwarg, ['b']), (arg_emptyvarargs_kwarg_no_default, ['b']) ]) -def test_GWArgSpec_keyword_args_correct_names(func, names): - spec = GWArgSpec(func) +def test_GWFuncDesc_keyword_args_correct_names(func, names): + spec = GWFuncDesc(func) assert len(spec.keyword_args) == len(names) for p, n in zip(spec.keyword_args, names): assert isinstance(p, KWArg) @@ -84,8 +83,8 @@ def test_GWArgSpec_keyword_args_correct_names(func, names): (arg_varargs_kwarg, ['test']), (arg_emptyvarargs_kwarg, ['test']) ]) -def test_GWArgSpec_keyword_args_have_defaults(func, defaults): - spec = GWArgSpec(func) +def test_GWFuncDesc_keyword_args_have_defaults(func, defaults): + spec = GWFuncDesc(func) assert len(spec.keyword_args) == len(defaults) for p, d in zip(spec.keyword_args, defaults): assert hasattr(p, 'default') @@ -95,8 +94,8 @@ def test_GWArgSpec_keyword_args_have_defaults(func, defaults): arg_varargs_kwarg_no_default, arg_emptyvarargs_kwarg_no_default ]) -def test_GWArgSpec_keyword_args_with_no_defaults_have_no_defaults(func): - spec = GWArgSpec(func) +def test_GWFuncDesc_keyword_args_with_no_defaults_have_no_defaults(func): + spec = GWFuncDesc(func) for p in spec.keyword_args: assert not hasattr(p, "default") @@ -104,8 +103,8 @@ def test_GWArgSpec_keyword_args_with_no_defaults_have_no_defaults(func): @pytest.mark.parametrize('func,data_types', [ (arg_with_annotation, [int]) ]) -def test_GWArgSpec_positional_args_with_annotation_have_data_type(func, data_types): - spec = GWArgSpec(func) +def test_GWFuncDesc_positional_args_with_annotation_have_data_type(func, data_types): + spec = GWFuncDesc(func) assert len(spec.positional_args) == len(data_types) for p, d in zip(spec.positional_args, data_types): assert hasattr(p, "data_type") From 4d274010aa4693a8caf7d6a6f078b596922ba612 Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Wed, 10 Oct 2018 10:04:43 -0400 Subject: [PATCH 03/12] Remove metadata checks from arguments Handling instance bound variables (i.e. 'self') will have to be solved in the CLI. --- girder_worker_utils/decorators.py | 8 +++----- girder_worker_utils/tests/decorators_test.py | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index e8248b4..c6d8466 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -140,21 +140,19 @@ def arguments(self): return [ self._construct_argument( self._get_class(self._signature.parameters[name]), name) - for name in self._signature.parameters if name in self._metadata] + for name in self._signature.parameters] @property def varargs(self): for name in self._signature.parameters: - if name in self._metadata and \ - self._is_varargs(self._signature.parameters[name]): + if self._is_varargs(self._signature.parameters[name]): return self._construct_argument(Varargs, name) return None @property def kwargs(self): for name in self._signature.parameters: - if name in self._metadata and \ - self._is_kwargs(self._signature.parameters[name]): + if self._is_kwargs(self._signature.parameters[name]): return self._construct_argument(Kwargs, name) return None diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index d9cc530..cd15cb8 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -302,7 +302,7 @@ def test_GWFuncDesc_keyword_args_have_defaults(func, defaults): assert hasattr(p, 'default') assert p.default == d - +@pytest.mark.skip("Fix this to use API for accessing argument spec rather than using 'private' attribute") def test_parameter_decorator_adds_metadata(): @parameter('a', test='TEST') def arg(a): From 6f2d578b9cd92139cad29187ab4622d3857dcd9c Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Wed, 10 Oct 2018 11:00:25 -0400 Subject: [PATCH 04/12] Fix linting and other code formatting issues --- girder_worker_utils/decorators.py | 41 +++++++++++-------- girder_worker_utils/tests/decorators_test.py | 31 +++++++------- .../tests/py3_decorators_test.py | 17 ++++---- 3 files changed, 48 insertions(+), 41 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index c6d8466..3f045fd 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -1,4 +1,4 @@ -from inspect import getdoc, cleandoc +from inspect import getdoc try: from inspect import signature, Parameter except ImportError: # pragma: nocover @@ -30,12 +30,23 @@ def __init__(self, name, **kwargs): for k, v in six.iteritems(kwargs): setattr(self, k, v) + # No default value for this argument -class Arg(Argument): pass +class Arg(Argument): + pass + + # Has a default argument for the value -class KWArg(Argument): pass -class Varargs(Argument): pass -class Kwargs(Argument): pass +class KWArg(Argument): + pass + + +class Varargs(Argument): + pass + + +class Kwargs(Argument): + pass # class Return(Argument): pass @@ -43,8 +54,6 @@ def _clean_function_doc(f): doc = getdoc(f) or '' if isinstance(doc, bytes): doc = doc.decode('utf-8') - else: - doc = cleandoc(doc) return doc @@ -81,7 +90,7 @@ def __getitem__(self, key): return self._construct_argument( self._get_class(self._signature.parameters[key]), key) - def _construct_argument(self, cls, name, **kwargs): + def _construct_argument(self, parameter_cls, name): p = self._signature.parameters[name] metadata = {} @@ -93,7 +102,7 @@ def _construct_argument(self, cls, name, **kwargs): metadata.update(self._metadata.get(name, {})) - return cls(name, **metadata) + return parameter_cls(name, **metadata) def _is_varargs(self, p): return p.kind == Parameter.VAR_POSITIONAL @@ -121,22 +130,20 @@ def _get_class(self, p): else: raise RuntimeError("Could not determine parameter type!") + def init_metadata(self, name): + if name not in self._metadata: + self._metadata[name] = {} def set_metadata(self, name, key, value): if name not in self._signature.parameters: raise RuntimeError("{} is not a valid argument to this function!") - if name not in self._metadata: - self._metadata[name] = {} + self.init_metadata(name) self._metadata[name][key] = value @property def arguments(self): - # Only return arguments if we've declared them as paramaters - # This prevents us from returning things like 'self' of bound - # methods (e.g. celery tasks) etc. This is a dubious design - # decision. return [ self._construct_argument( self._get_class(self._signature.parameters[name]), name) @@ -170,7 +177,7 @@ def parameter(name, **kwargs): raise TypeError('Expected argument name to be a string') data_type = kwargs.get("data_type", None) - if data_type is not None and callable(data_type): + if callable(data_type): kwargs['data_type'] = data_type(name, **kwargs) def argument_wrapper(func): @@ -184,7 +191,7 @@ def argument_wrapper(func): # the full list of parameters that have been identified by the # user (even if there is no actual metadata associated with # the argument). - desc._metadata[name] = {} + desc.init_metadata(name) for key, value in six.iteritems(kwargs): desc.set_metadata(name, key, value) diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index cd15cb8..eaae61e 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -2,7 +2,14 @@ from girder_worker_utils import decorators from girder_worker_utils import types -from girder_worker_utils.decorators import argument +from girder_worker_utils.decorators import ( + argument, + parameter, + GWFuncDesc, + Varargs, + Kwargs, + Arg, + KWArg) @argument('n', types.Integer, help='The element to return') @@ -173,22 +180,8 @@ def test_unhandled_input_binding(): with pytest.raises(ValueError): decorators.get_input_data(arg, {}) - - ########################### - -import six - -from girder_worker_utils.decorators import parameter -from girder_worker_utils.decorators import ( - GWFuncDesc, - Varargs, - Kwargs, - Arg, - KWArg) - - def arg(a): pass # noqa def varargs(*args): pass # noqa def kwarg(a='test'): pass # noqa @@ -205,7 +198,6 @@ def arg_kwarg_kwargs(a, b='test', **kwargs): pass # noqa def arg_kwarg_varargs_kwargs(a, b='test', *args, **kwargs): pass # noqa - @pytest.mark.parametrize('func,classes', [ (arg, [Arg]), (varargs, [Varargs]), @@ -233,6 +225,7 @@ def test_GWFuncDesc_arguments_returns_expected_classes(func, classes): arg_kwargs, kwarg_kwarg, kwarg_kwargs, arg_kwarg_kwargs] + @pytest.mark.parametrize('func', no_varargs) def test_GWFuncDesc_varargs_returns_None(func): spec = GWFuncDesc(func) @@ -242,6 +235,7 @@ def test_GWFuncDesc_varargs_returns_None(func): with_varargs = [varargs, arg_varargs, kwarg_varargs, arg_kwarg_varargs, arg_kwarg_varargs_kwargs] + @pytest.mark.parametrize('func', with_varargs) def test_GWFuncDesc_varargs_returns_Vararg(func): spec = GWFuncDesc(func) @@ -284,6 +278,7 @@ def test_GWFuncDesc_keyword_args_correct_names(func, names): assert isinstance(p, KWArg) assert p.name == n + # TODO keyword_args returns None test @pytest.mark.parametrize('func,defaults', [ (kwarg, ['test']), @@ -302,7 +297,9 @@ def test_GWFuncDesc_keyword_args_have_defaults(func, defaults): assert hasattr(p, 'default') assert p.default == d -@pytest.mark.skip("Fix this to use API for accessing argument spec rather than using 'private' attribute") + +@pytest.mark.skip("Fix this to use API for accessing argument " + "spec rather than using 'private' attribute") def test_parameter_decorator_adds_metadata(): @parameter('a', test='TEST') def arg(a): diff --git a/girder_worker_utils/tests/py3_decorators_test.py b/girder_worker_utils/tests/py3_decorators_test.py index fc614f4..306632c 100644 --- a/girder_worker_utils/tests/py3_decorators_test.py +++ b/girder_worker_utils/tests/py3_decorators_test.py @@ -1,17 +1,18 @@ import pytest -import six from girder_worker_utils.decorators import ( GWFuncDesc, Varargs, - Kwargs, + # Kwargs, Arg, KWArg) -def arg_varargs_kwarg(a, *args, b='test'): pass -def arg_varargs_kwarg_no_default(a, *args, b): pass -def arg_emptyvarargs_kwarg(a, *, b='test'): pass -def arg_emptyvarargs_kwarg_no_default(a, *, b): pass -def arg_with_annotation(a: int): pass +# TODO write Kwargs test + +def arg_varargs_kwarg(a, *args, b='test'): pass # noqa +def arg_varargs_kwarg_no_default(a, *args, b): pass # noqa +def arg_emptyvarargs_kwarg(a, *, b='test'): pass # noqa +def arg_emptyvarargs_kwarg_no_default(a, *, b): pass # noqa +def arg_with_annotation(a: int): pass # noqa @pytest.mark.parametrize('func,classes', [ @@ -79,6 +80,7 @@ def test_GWFuncDesc_keyword_args_correct_names(func, names): # TODO keyword_args returns None test + @pytest.mark.parametrize('func,defaults', [ (arg_varargs_kwarg, ['test']), (arg_emptyvarargs_kwarg, ['test']) @@ -90,6 +92,7 @@ def test_GWFuncDesc_keyword_args_have_defaults(func, defaults): assert hasattr(p, 'default') assert p.default == d + @pytest.mark.parametrize('func', [ arg_varargs_kwarg_no_default, arg_emptyvarargs_kwarg_no_default From 672d1f419880613de54920695566e6ef2a5c6799 Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Wed, 10 Oct 2018 14:07:59 -0400 Subject: [PATCH 05/12] Provide class level API for checking presence of GWFuncDesc --- girder_worker_utils/decorators.py | 25 +++++++++++++++----- girder_worker_utils/tests/decorators_test.py | 11 +++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index 3f045fd..6f4dac8 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -47,6 +47,9 @@ class Varargs(Argument): class Kwargs(Argument): pass + +# TODO: is there anything we want to try and do with the functions +# annotated return value? # class Return(Argument): pass @@ -67,13 +70,23 @@ class GWFuncDesc(object): @classmethod def get_description(cls, func): - # HACK - potentially unwrap celery task - # func = getattr(func, 'run', func) - if hasattr(func, cls._func_desc_attr) and \ + if cls.has_description(func) and \ isinstance(getattr(func, cls._func_desc_attr), cls): return getattr(func, cls._func_desc_attr) return None + @classmethod + def has_description(cls, func): + return hasattr(func, cls._func_desc_attr) + + @classmethod + def set_description(cls, func): + setattr(func, GWFuncDesc._func_desc_attr, cls(func)) + return None + + + + def __init__(self, func): self.func_name = func.__name__ self.func_help = _clean_function_doc(func) @@ -181,10 +194,10 @@ def parameter(name, **kwargs): kwargs['data_type'] = data_type(name, **kwargs) def argument_wrapper(func): - if not hasattr(func, GWFuncDesc._func_desc_attr): - setattr(func, GWFuncDesc._func_desc_attr, GWFuncDesc(func)) + if not GWFuncDesc.has_description(func): + GWFuncDesc.set_description(func) - desc = getattr(func, GWFuncDesc._func_desc_attr) + desc = GWFuncDesc.get_description(func) # Make sure the metadata key exists even if we don't set any # values on it. This ensures that metadata's keys represent diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index eaae61e..19f1669 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -298,12 +298,15 @@ def test_GWFuncDesc_keyword_args_have_defaults(func, defaults): assert p.default == d -@pytest.mark.skip("Fix this to use API for accessing argument " - "spec rather than using 'private' attribute") def test_parameter_decorator_adds_metadata(): @parameter('a', test='TEST') def arg(a): pass - assert hasattr(arg._girder_spec['a'], 'test') - assert arg._girder_spec['a'].test == 'TEST' + desc = GWFuncDesc.get_description(arg) + + assert hasattr(desc.arguments[0], 'test') + assert desc.arguments[0].test == 'TEST' + + +# TODO more tests around other argument types (e.g. kwargs varargs etc) From 237cf79bb6bdfd342bb08ed51b05189ee843bb3e Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Thu, 11 Oct 2018 11:06:58 -0400 Subject: [PATCH 06/12] Remove unnecessary compatibility code from parameter decorator --- girder_worker_utils/decorators.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index 6f4dac8..190537c 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -209,11 +209,6 @@ def argument_wrapper(func): for key, value in six.iteritems(kwargs): desc.set_metadata(name, key, value) - def description(): - return getattr(func, GWFuncDesc._func_desc_attr) - - func.description = description - return func return argument_wrapper From 519fe45d942f7a7546907a9f867ecb7396142685 Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Thu, 11 Oct 2018 11:21:09 -0400 Subject: [PATCH 07/12] Refactor GWFuncDesc __repr__ to be less confusing --- girder_worker_utils/decorators.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index 190537c..49acff7 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -84,9 +84,6 @@ def set_description(cls, func): setattr(func, GWFuncDesc._func_desc_attr, cls(func)) return None - - - def __init__(self, func): self.func_name = func.__name__ self.func_help = _clean_function_doc(func) @@ -94,10 +91,12 @@ def __init__(self, func): self._signature = signature(func) def __repr__(self): - # TODO - make less ugly - return "<{}(".format(self.__class__.__name__) + ", ".join(["{}:{}".format( - name, self._parameter_repr[self._signature.parameters[name].kind]) - for name in self._signature.parameters]) + ")>" + parameters = [] + for name in self._signature.parameters: + kind = self._signature.parameters[name].kind + parameters.append("{}:{}".format(name, self._parameter_repr[kind])) + + return "<{}(".format(self.__class__.__name__) + ", ".join(parameters) + ")>" def __getitem__(self, key): return self._construct_argument( From 11c610245fcf40cd6e6f5528c555844efef8b6ec Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Thu, 11 Oct 2018 11:38:56 -0400 Subject: [PATCH 08/12] Add tests for positional/keyword args if none exist in signature --- girder_worker_utils/tests/decorators_test.py | 18 ++++++++++++++++-- .../tests/py3_decorators_test.py | 7 +------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index 19f1669..a2e85d8 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -259,7 +259,14 @@ def test_GWFuncDesc_positional_args_correct_names(func, names): assert p.name == n -# TODO positional_args returns None test +@pytest.mark.parametrize('func', [ + varargs, kwarg, kwargs, kwarg_varargs, + kwarg_kwarg, kwarg_kwargs +]) +def test_GWFuncDesc_positional_args_returns_empty_list_if_no_positional_args(func): + spec = GWFuncDesc(func) + assert len(spec.positional_args) == 0 + @pytest.mark.parametrize('func,names', [ (kwarg, ['a']), @@ -279,7 +286,14 @@ def test_GWFuncDesc_keyword_args_correct_names(func, names): assert p.name == n -# TODO keyword_args returns None test +@pytest.mark.parametrize('func', [ + arg, varargs, kwargs, arg_arg, arg_varargs, arg_kwargs +]) +def test_GWFuncDesc_keyword_args_returns_empty_list_if_no_keyword_args(func): + spec = GWFuncDesc(func) + assert len(spec.keyword_args) == 0 + + @pytest.mark.parametrize('func,defaults', [ (kwarg, ['test']), (arg_kwarg, ['test']), diff --git a/girder_worker_utils/tests/py3_decorators_test.py b/girder_worker_utils/tests/py3_decorators_test.py index 306632c..c01e7e9 100644 --- a/girder_worker_utils/tests/py3_decorators_test.py +++ b/girder_worker_utils/tests/py3_decorators_test.py @@ -1,12 +1,11 @@ import pytest + from girder_worker_utils.decorators import ( GWFuncDesc, Varargs, - # Kwargs, Arg, KWArg) -# TODO write Kwargs test def arg_varargs_kwarg(a, *args, b='test'): pass # noqa def arg_varargs_kwarg_no_default(a, *args, b): pass # noqa @@ -63,8 +62,6 @@ def test_GWFuncDesc_positional_args_correct_names(func, names): assert p.name == n -# TODO positional_args returns None test - @pytest.mark.parametrize('func,names', [ (arg_varargs_kwarg, ['b']), (arg_varargs_kwarg_no_default, ['b']), @@ -78,8 +75,6 @@ def test_GWFuncDesc_keyword_args_correct_names(func, names): assert isinstance(p, KWArg) assert p.name == n -# TODO keyword_args returns None test - @pytest.mark.parametrize('func,defaults', [ (arg_varargs_kwarg, ['test']), From ac4755d67403cbf33a89147b974c5181baac69b8 Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Wed, 24 Oct 2018 10:29:27 -0400 Subject: [PATCH 09/12] Rename GWFuncDesc Argument class names --- girder_worker_utils/decorators.py | 24 +++++----- girder_worker_utils/tests/decorators_test.py | 44 +++++++++---------- .../tests/py3_decorators_test.py | 20 ++++----- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index 49acff7..5fd6037 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -32,20 +32,20 @@ def __init__(self, name, **kwargs): # No default value for this argument -class Arg(Argument): +class PositionalArg(Argument): pass # Has a default argument for the value -class KWArg(Argument): +class KeywordArg(Argument): pass -class Varargs(Argument): +class VarsArg(Argument): pass -class Kwargs(Argument): +class KwargsArg(Argument): pass # TODO: is there anything we want to try and do with the functions @@ -132,13 +132,13 @@ def _is_posarg(self, p): def _get_class(self, p): if self._is_varargs(p): - return Varargs + return VarsArg elif self._is_kwargs(p): - return Kwargs + return KwargsArg elif self._is_posarg(p): - return Arg + return PositionalArg elif self._is_kwarg(p): - return KWArg + return KeywordArg else: raise RuntimeError("Could not determine parameter type!") @@ -165,23 +165,23 @@ def arguments(self): def varargs(self): for name in self._signature.parameters: if self._is_varargs(self._signature.parameters[name]): - return self._construct_argument(Varargs, name) + return self._construct_argument(VarsArg, name) return None @property def kwargs(self): for name in self._signature.parameters: if self._is_kwargs(self._signature.parameters[name]): - return self._construct_argument(Kwargs, name) + return self._construct_argument(KeywordArg, name) return None @property def positional_args(self): - return [arg for arg in self.arguments if isinstance(arg, Arg)] + return [arg for arg in self.arguments if isinstance(arg, PositionalArg)] @property def keyword_args(self): - return [arg for arg in self.arguments if isinstance(arg, KWArg)] + return [arg for arg in self.arguments if isinstance(arg, KeywordArg)] def parameter(name, **kwargs): diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index a2e85d8..3e6412a 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -4,12 +4,12 @@ from girder_worker_utils import types from girder_worker_utils.decorators import ( argument, - parameter, GWFuncDesc, - Varargs, - Kwargs, - Arg, - KWArg) + KeywordArg, + KwargsArg, + parameter, + PositionalArg, + VarsArg) @argument('n', types.Integer, help='The element to return') @@ -199,20 +199,20 @@ def arg_kwarg_varargs_kwargs(a, b='test', *args, **kwargs): pass # noqa @pytest.mark.parametrize('func,classes', [ - (arg, [Arg]), - (varargs, [Varargs]), - (kwarg, [KWArg]), - (kwargs, [Kwargs]), - (arg_arg, [Arg, Arg]), - (arg_varargs, [Arg, Varargs]), - (arg_kwarg, [Arg, KWArg]), - (arg_kwargs, [Arg, Kwargs]), - (kwarg_varargs, [KWArg, Varargs]), - (kwarg_kwarg, [KWArg, KWArg]), - (kwarg_kwargs, [KWArg, Kwargs]), - (arg_kwarg_varargs, [Arg, KWArg, Varargs]), - (arg_kwarg_kwargs, [Arg, KWArg, Kwargs]), - (arg_kwarg_varargs_kwargs, [Arg, KWArg, Varargs, Kwargs]) + (arg, [PositionalArg]), + (varargs, [VarsArg]), + (kwarg, [KeywordArg]), + (kwargs, [KwargsArg]), + (arg_arg, [PositionalArg, PositionalArg]), + (arg_varargs, [PositionalArg, VarsArg]), + (arg_kwarg, [PositionalArg, KeywordArg]), + (arg_kwargs, [PositionalArg, KwargsArg]), + (kwarg_varargs, [KeywordArg, VarsArg]), + (kwarg_kwarg, [KeywordArg, KeywordArg]), + (kwarg_kwargs, [KeywordArg, KwargsArg]), + (arg_kwarg_varargs, [PositionalArg, KeywordArg, VarsArg]), + (arg_kwarg_kwargs, [PositionalArg, KeywordArg, KwargsArg]), + (arg_kwarg_varargs_kwargs, [PositionalArg, KeywordArg, VarsArg, KwargsArg]) ]) def test_GWFuncDesc_arguments_returns_expected_classes(func, classes): spec = GWFuncDesc(func) @@ -239,7 +239,7 @@ def test_GWFuncDesc_varargs_returns_None(func): @pytest.mark.parametrize('func', with_varargs) def test_GWFuncDesc_varargs_returns_Vararg(func): spec = GWFuncDesc(func) - assert isinstance(spec.varargs, Varargs) + assert isinstance(spec.varargs, VarsArg) @pytest.mark.parametrize('func,names', [ @@ -255,7 +255,7 @@ def test_GWFuncDesc_positional_args_correct_names(func, names): spec = GWFuncDesc(func) assert len(spec.positional_args) == len(names) for p, n in zip(spec.positional_args, names): - assert isinstance(p, Arg) + assert isinstance(p, PositionalArg) assert p.name == n @@ -282,7 +282,7 @@ def test_GWFuncDesc_keyword_args_correct_names(func, names): spec = GWFuncDesc(func) assert len(spec.keyword_args) == len(names) for p, n in zip(spec.keyword_args, names): - assert isinstance(p, KWArg) + assert isinstance(p, KeywordArg) assert p.name == n diff --git a/girder_worker_utils/tests/py3_decorators_test.py b/girder_worker_utils/tests/py3_decorators_test.py index c01e7e9..d5d59d8 100644 --- a/girder_worker_utils/tests/py3_decorators_test.py +++ b/girder_worker_utils/tests/py3_decorators_test.py @@ -2,9 +2,9 @@ from girder_worker_utils.decorators import ( GWFuncDesc, - Varargs, - Arg, - KWArg) + KeywordArg, + PositionalArg, + VarsArg) def arg_varargs_kwarg(a, *args, b='test'): pass # noqa @@ -15,10 +15,10 @@ def arg_with_annotation(a: int): pass # noqa @pytest.mark.parametrize('func,classes', [ - (arg_varargs_kwarg, [Arg, Varargs, KWArg]), - (arg_varargs_kwarg_no_default, [Arg, Varargs, KWArg]), - (arg_emptyvarargs_kwarg, [Arg, KWArg]), - (arg_emptyvarargs_kwarg_no_default, [Arg, KWArg]) + (arg_varargs_kwarg, [PositionalArg, VarsArg, KeywordArg]), + (arg_varargs_kwarg_no_default, [PositionalArg, VarsArg, KeywordArg]), + (arg_emptyvarargs_kwarg, [PositionalArg, KeywordArg]), + (arg_emptyvarargs_kwarg_no_default, [PositionalArg, KeywordArg]) ]) def test_GWFuncDesc_arguments_returns_expected_classes(func, classes): spec = GWFuncDesc(func) @@ -45,7 +45,7 @@ def test_GWFuncDesc_varargs_returns_None(func): @pytest.mark.parametrize('func', with_varargs) def test_GWFuncDesc_varargs_returns_Vararg(func): spec = GWFuncDesc(func) - assert isinstance(spec.varargs, Varargs) + assert isinstance(spec.varargs, VarsArg) @pytest.mark.parametrize('func,names', [ @@ -58,7 +58,7 @@ def test_GWFuncDesc_positional_args_correct_names(func, names): spec = GWFuncDesc(func) assert len(spec.positional_args) == len(names) for p, n in zip(spec.positional_args, names): - assert isinstance(p, Arg) + assert isinstance(p, PositionalArg) assert p.name == n @@ -72,7 +72,7 @@ def test_GWFuncDesc_keyword_args_correct_names(func, names): spec = GWFuncDesc(func) assert len(spec.keyword_args) == len(names) for p, n in zip(spec.keyword_args, names): - assert isinstance(p, KWArg) + assert isinstance(p, KeywordArg) assert p.name == n From 9999eba35e1a19f146dce76cb23c67c7b018734d Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Wed, 24 Oct 2018 11:23:59 -0400 Subject: [PATCH 10/12] Fix bad copy-pasta, this is definitely Alpha --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6688897..b8ff2b2 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def prerelease_local_scheme(version): author_email='kitware@kitware.com', license='Apache 2.0', classifiers=[ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 3 - Alpha', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python', From 99cd62d9aeb5e92bf3aaa032cf824d4b42e45f4a Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Wed, 24 Oct 2018 11:26:14 -0400 Subject: [PATCH 11/12] Add deprecation, deprecate the argument decorator --- girder_worker_utils/decorators.py | 8 ++ girder_worker_utils/tests/decorators_test.py | 77 +++++++++++--------- requirements.in | 1 + requirements.txt | 6 +- 4 files changed, 57 insertions(+), 35 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index 5fd6037..4646df2 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -1,4 +1,9 @@ from inspect import getdoc + +import deprecation + +from girder_worker_utils import __version__ + try: from inspect import signature, Parameter except ImportError: # pragma: nocover @@ -213,6 +218,9 @@ def argument_wrapper(func): return argument_wrapper +@deprecation.deprecated(deprecated_in="0.8.5", removed_in="0.9.0", + current_version=__version__, + details="Use 'parameter' decorator instead") def argument(name, data_type, *args, **kwargs): """Describe an argument to a function as a function decorator. diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index 3e6412a..a71251a 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -1,3 +1,4 @@ +import deprecation import pytest from girder_worker_utils import decorators @@ -12,42 +13,16 @@ VarsArg) -@argument('n', types.Integer, help='The element to return') -def fibonacci(n): - """Compute a fibonacci number.""" - if n <= 2: - return 1 - return fibonacci(n - 1) + fibonacci(n - 2) - - -@argument('val', types.String, help='The value to return') -def keyword_func(val='test'): - """Return a value.""" - return val - - -@argument('arg1', types.String) -@argument('arg2', types.StringChoice, choices=('a', 'b')) -@argument('kwarg1', types.StringVector) -@argument('kwarg2', types.Number, min=0, max=10) -@argument('kwarg3', types.NumberMultichoice, choices=(1, 2, 3, 4, 5)) -def complex_func(arg1, arg2, kwarg1=('one',), kwarg2=4, kwarg3=(1, 2)): - return { - 'arg1': arg1, - 'arg2': arg2, - 'kwarg1': kwarg1, - 'kwarg2': kwarg2, - 'kwarg3': kwarg3 - } - - -@argument('item', types.GirderItem) -@argument('folder', types.GirderFolder) -def girder_types_func(item, folder): - return item, folder +@deprecation.fail_if_not_removed +def test_positional_argument(): + @argument('n', types.Integer, help='The element to return') + def fibonacci(n): + """Compute a fibonacci number.""" + if n <= 2: + return 1 + return fibonacci(n - 1) + fibonacci(n - 2) -def test_positional_argument(): desc = fibonacci.describe() assert len(desc['inputs']) == 1 assert desc['name'].split('.')[-1] == 'fibonacci' @@ -63,7 +38,14 @@ def test_positional_argument(): fibonacci.call_item_task({}) +@deprecation.fail_if_not_removed def test_keyword_argument(): + + @argument('val', types.String, help='The value to return') + def keyword_func(val='test'): + """Return a value.""" + return val + desc = keyword_func.describe() assert len(desc['inputs']) == 1 assert desc['name'].split('.')[-1] == 'keyword_func' @@ -78,7 +60,23 @@ def test_keyword_argument(): assert keyword_func.call_item_task({}) == 'test' +@deprecation.fail_if_not_removed def test_multiple_arguments(): + + @argument('arg1', types.String) + @argument('arg2', types.StringChoice, choices=('a', 'b')) + @argument('kwarg1', types.StringVector) + @argument('kwarg2', types.Number, min=0, max=10) + @argument('kwarg3', types.NumberMultichoice, choices=(1, 2, 3, 4, 5)) + def complex_func(arg1, arg2, kwarg1=('one',), kwarg2=4, kwarg3=(1, 2)): + return { + 'arg1': arg1, + 'arg2': arg2, + 'kwarg1': kwarg1, + 'kwarg2': kwarg2, + 'kwarg3': kwarg3 + } + desc = complex_func.describe() assert len(desc['inputs']) == 5 assert desc['name'].split('.')[-1] == 'complex_func' @@ -136,7 +134,14 @@ def test_multiple_arguments(): } +@deprecation.fail_if_not_removed def test_girder_input_mode(): + + @argument('item', types.GirderItem) + @argument('folder', types.GirderFolder) + def girder_types_func(item, folder): + return item, folder + item, folder = girder_types_func.call_item_task({ 'item': { 'mode': 'girder', @@ -155,6 +160,7 @@ def test_girder_input_mode(): assert folder == 'folderid' +@deprecation.fail_if_not_removed def test_missing_description_exception(): def func(): pass @@ -163,11 +169,13 @@ def func(): decorators.get_description_attribute(func) +@deprecation.fail_if_not_removed def test_argument_name_not_string(): with pytest.raises(TypeError): argument(0, types.Integer) +@deprecation.fail_if_not_removed def test_argument_name_not_a_parameter(): with pytest.raises(ValueError): @argument('notarg', types.Integer) @@ -175,6 +183,7 @@ def func(arg): pass +@deprecation.fail_if_not_removed def test_unhandled_input_binding(): arg = argument('arg', types.Integer) with pytest.raises(ValueError): diff --git a/requirements.in b/requirements.in index 23a8c80..223976a 100644 --- a/requirements.in +++ b/requirements.in @@ -3,3 +3,4 @@ girder-client>=2 jsonpickle setuptools_scm six +deprecation diff --git a/requirements.txt b/requirements.txt index 8dce4b5..720a3b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,26 +8,30 @@ certifi==2017.7.27.1 # via requests chardet==3.0.4 # via requests click==6.7 # via girder-client coverage==4.4.1 # via pytest-cov +deprecation==2.0.6 diskcache==2.9.0 # via girder-client flake8-blind-except==0.1.1 flake8-docstrings==1.1.0 flake8-import-order==0.13 flake8-polyfill==1.0.1 # via flake8-docstrings flake8==3.4.1 -funcsigs==1.0.2 ; python_version < "3.5" girder-client==2.3.0 idna==2.6 # via requests +jsonpickle==1.0 mccabe==0.6.1 # via flake8 mock==2.0.0 +packaging==18.0 # via deprecation pbr==3.1.1 # via mock py==1.4.34 # via pytest pycodestyle==2.3.1 # via flake8, flake8-import-order pydocstyle==2.0.0 # via flake8-docstrings pyflakes==1.5.0 # via flake8 +pyparsing==2.2.2 # via packaging pytest-cov==2.5.1 pytest==3.2.3 requests-toolbelt==0.8.0 # via girder-client requests==2.18.4 # via girder-client, requests-toolbelt +setuptools-scm==3.1.0 six==1.11.0 snowballstemmer==1.2.1 # via pydocstyle urllib3==1.22 # via requests From 3d1d25512a07e8321815910fce093c0986e7292c Mon Sep 17 00:00:00 2001 From: Christopher Kotfila Date: Thu, 1 Nov 2018 16:40:23 -0400 Subject: [PATCH 12/12] Allow GWArgSpec consumers to define Arg class types This allows metadata mapping for each type of argument to be defined on the consumers' (e.g. CLI) classes --- girder_worker_utils/decorators.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index 4646df2..af7d338 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -73,6 +73,11 @@ class GWFuncDesc(object): 'KEYWORD_ONLY', 'VAR_KEYWORD'] + VarsArgCls = VarsArg + KwargsArgCls = KwargsArg + PositionalArgCls = PositionalArg + KeywordArgCls = KeywordArg + @classmethod def get_description(cls, func): if cls.has_description(func) and \ @@ -137,13 +142,13 @@ def _is_posarg(self, p): def _get_class(self, p): if self._is_varargs(p): - return VarsArg + return self.VarsArgCls elif self._is_kwargs(p): - return KwargsArg + return self.KwargsArgCls elif self._is_posarg(p): - return PositionalArg + return self.PositionalArgCls elif self._is_kwarg(p): - return KeywordArg + return self.KeywordArgCls else: raise RuntimeError("Could not determine parameter type!") @@ -153,7 +158,7 @@ def init_metadata(self, name): def set_metadata(self, name, key, value): if name not in self._signature.parameters: - raise RuntimeError("{} is not a valid argument to this function!") + raise RuntimeError("{} is not a valid argument to this function!".format(name)) self.init_metadata(name)