diff --git a/girder_worker_utils/decorators.py b/girder_worker_utils/decorators.py index ca21847..af7d338 100644 --- a/girder_worker_utils/decorators.py +++ b/girder_worker_utils/decorators.py @@ -1,8 +1,14 @@ from inspect import getdoc + +import deprecation + +from girder_worker_utils import __version__ + try: - from inspect import signature + from inspect import signature, Parameter except ImportError: # pragma: nocover - from funcsigs import signature + from funcsigs import signature, Parameter + import six @@ -16,13 +22,210 @@ class MissingInputException(Exception): def get_description_attribute(func): """Get the private description attribute from a function.""" - func = getattr(func, 'run', func) - description = getattr(func, '_girder_description', None) + # func = getattr(func, 'run', func) + description = getattr(func, GWFuncDesc._func_desc_attr, None) if description is None: raise MissingDescriptionException('Function is missing description decorators') return description +class Argument(object): + def __init__(self, name, **kwargs): + self.name = name + for k, v in six.iteritems(kwargs): + setattr(self, k, v) + + +# No default value for this argument +class PositionalArg(Argument): + pass + + +# Has a default argument for the value +class KeywordArg(Argument): + pass + + +class VarsArg(Argument): + pass + + +class KwargsArg(Argument): + pass + +# TODO: is there anything we want to try and do with the functions +# annotated return value? +# class Return(Argument): pass + + +def _clean_function_doc(f): + doc = getdoc(f) or '' + if isinstance(doc, bytes): + doc = doc.decode('utf-8') + return doc + + +class GWFuncDesc(object): + _func_desc_attr = "_gw_function_description" + _parameter_repr = ['POSITIONAL_ONLY', + 'POSITIONAL_OR_KEYWORD', + 'VAR_POSITIONAL', + 'KEYWORD_ONLY', + 'VAR_KEYWORD'] + + VarsArgCls = VarsArg + KwargsArgCls = KwargsArg + PositionalArgCls = PositionalArg + KeywordArgCls = KeywordArg + + @classmethod + def get_description(cls, func): + if cls.has_description(func) and \ + isinstance(getattr(func, cls._func_desc_attr), cls): + return getattr(func, cls._func_desc_attr) + return None + + @classmethod + def has_description(cls, func): + return hasattr(func, cls._func_desc_attr) + + @classmethod + def set_description(cls, func): + setattr(func, GWFuncDesc._func_desc_attr, cls(func)) + return None + + def __init__(self, func): + self.func_name = func.__name__ + self.func_help = _clean_function_doc(func) + self._metadata = {} + self._signature = signature(func) + + def __repr__(self): + parameters = [] + for name in self._signature.parameters: + kind = self._signature.parameters[name].kind + parameters.append("{}:{}".format(name, self._parameter_repr[kind])) + + return "<{}(".format(self.__class__.__name__) + ", ".join(parameters) + ")>" + + def __getitem__(self, key): + return self._construct_argument( + self._get_class(self._signature.parameters[key]), key) + + def _construct_argument(self, parameter_cls, name): + p = self._signature.parameters[name] + metadata = {} + + if p.default != p.empty: + metadata['default'] = p.default + if p.annotation != p.empty: + # TODO: make sure annotation is a type and not just garbage + metadata['data_type'] = p.annotation + + metadata.update(self._metadata.get(name, {})) + + return parameter_cls(name, **metadata) + + def _is_varargs(self, p): + return p.kind == Parameter.VAR_POSITIONAL + + def _is_kwargs(self, p): + return p.kind == Parameter.VAR_KEYWORD + + def _is_kwarg(self, p): + return p.kind == Parameter.KEYWORD_ONLY or ( + p.kind == Parameter.POSITIONAL_OR_KEYWORD and p.default != p.empty) + + def _is_posarg(self, p): + return p.kind == Parameter.POSITIONAL_ONLY or ( + p.kind == Parameter.POSITIONAL_OR_KEYWORD and p.default == p.empty) + + def _get_class(self, p): + if self._is_varargs(p): + return self.VarsArgCls + elif self._is_kwargs(p): + return self.KwargsArgCls + elif self._is_posarg(p): + return self.PositionalArgCls + elif self._is_kwarg(p): + return self.KeywordArgCls + else: + raise RuntimeError("Could not determine parameter type!") + + def init_metadata(self, name): + if name not in self._metadata: + self._metadata[name] = {} + + def set_metadata(self, name, key, value): + if name not in self._signature.parameters: + raise RuntimeError("{} is not a valid argument to this function!".format(name)) + + self.init_metadata(name) + + self._metadata[name][key] = value + + @property + def arguments(self): + return [ + self._construct_argument( + self._get_class(self._signature.parameters[name]), name) + for name in self._signature.parameters] + + @property + def varargs(self): + for name in self._signature.parameters: + if self._is_varargs(self._signature.parameters[name]): + return self._construct_argument(VarsArg, name) + return None + + @property + def kwargs(self): + for name in self._signature.parameters: + if self._is_kwargs(self._signature.parameters[name]): + return self._construct_argument(KeywordArg, name) + return None + + @property + def positional_args(self): + return [arg for arg in self.arguments if isinstance(arg, PositionalArg)] + + @property + def keyword_args(self): + return [arg for arg in self.arguments if isinstance(arg, KeywordArg)] + + +def parameter(name, **kwargs): + if not isinstance(name, six.string_types): + raise TypeError('Expected argument name to be a string') + + data_type = kwargs.get("data_type", None) + if callable(data_type): + kwargs['data_type'] = data_type(name, **kwargs) + + def argument_wrapper(func): + if not GWFuncDesc.has_description(func): + GWFuncDesc.set_description(func) + + desc = GWFuncDesc.get_description(func) + + # Make sure the metadata key exists even if we don't set any + # values on it. This ensures that metadata's keys represent + # the full list of parameters that have been identified by the + # user (even if there is no actual metadata associated with + # the argument). + desc.init_metadata(name) + + for key, value in six.iteritems(kwargs): + desc.set_metadata(name, key, value) + + return func + + return argument_wrapper + + +@deprecation.deprecated(deprecated_in="0.8.5", removed_in="0.9.0", + current_version=__version__, + details="Use 'parameter' decorator instead") def argument(name, data_type, *args, **kwargs): """Describe an argument to a function as a function decorator. @@ -38,8 +241,10 @@ def argument(name, data_type, *args, **kwargs): data_type = data_type(name, *args, **kwargs) def argument_wrapper(func): - func._girder_description = getattr(func, '_girder_description', {}) - args = func._girder_description.setdefault('arguments', []) + setattr(func, GWFuncDesc._func_desc_attr, + getattr(func, GWFuncDesc._func_desc_attr, {})) + + args = getattr(func, GWFuncDesc._func_desc_attr).setdefault('arguments', []) sig = signature(func) if name not in sig.parameters: diff --git a/girder_worker_utils/tests/conftest.py b/girder_worker_utils/tests/conftest.py new file mode 100644 index 0000000..98fd7a6 --- /dev/null +++ b/girder_worker_utils/tests/conftest.py @@ -0,0 +1,5 @@ +import six + +collect_ignore = [] +if six.PY2: + collect_ignore.append("py3_decorators_test.py") diff --git a/girder_worker_utils/tests/decorators_test.py b/girder_worker_utils/tests/decorators_test.py index 85a9c78..a71251a 100644 --- a/girder_worker_utils/tests/decorators_test.py +++ b/girder_worker_utils/tests/decorators_test.py @@ -1,46 +1,28 @@ +import deprecation import pytest from girder_worker_utils import decorators from girder_worker_utils import types -from girder_worker_utils.decorators import argument - - -@argument('n', types.Integer, help='The element to return') -def fibonacci(n): - """Compute a fibonacci number.""" - if n <= 2: - return 1 - return fibonacci(n - 1) + fibonacci(n - 2) - - -@argument('val', types.String, help='The value to return') -def keyword_func(val='test'): - """Return a value.""" - return val - - -@argument('arg1', types.String) -@argument('arg2', types.StringChoice, choices=('a', 'b')) -@argument('kwarg1', types.StringVector) -@argument('kwarg2', types.Number, min=0, max=10) -@argument('kwarg3', types.NumberMultichoice, choices=(1, 2, 3, 4, 5)) -def complex_func(arg1, arg2, kwarg1=('one',), kwarg2=4, kwarg3=(1, 2)): - return { - 'arg1': arg1, - 'arg2': arg2, - 'kwarg1': kwarg1, - 'kwarg2': kwarg2, - 'kwarg3': kwarg3 - } +from girder_worker_utils.decorators import ( + argument, + GWFuncDesc, + KeywordArg, + KwargsArg, + parameter, + PositionalArg, + VarsArg) -@argument('item', types.GirderItem) -@argument('folder', types.GirderFolder) -def girder_types_func(item, folder): - return item, folder +@deprecation.fail_if_not_removed +def test_positional_argument(): + @argument('n', types.Integer, help='The element to return') + def fibonacci(n): + """Compute a fibonacci number.""" + if n <= 2: + return 1 + return fibonacci(n - 1) + fibonacci(n - 2) -def test_positional_argument(): desc = fibonacci.describe() assert len(desc['inputs']) == 1 assert desc['name'].split('.')[-1] == 'fibonacci' @@ -56,7 +38,14 @@ def test_positional_argument(): fibonacci.call_item_task({}) +@deprecation.fail_if_not_removed def test_keyword_argument(): + + @argument('val', types.String, help='The value to return') + def keyword_func(val='test'): + """Return a value.""" + return val + desc = keyword_func.describe() assert len(desc['inputs']) == 1 assert desc['name'].split('.')[-1] == 'keyword_func' @@ -71,7 +60,23 @@ def test_keyword_argument(): assert keyword_func.call_item_task({}) == 'test' +@deprecation.fail_if_not_removed def test_multiple_arguments(): + + @argument('arg1', types.String) + @argument('arg2', types.StringChoice, choices=('a', 'b')) + @argument('kwarg1', types.StringVector) + @argument('kwarg2', types.Number, min=0, max=10) + @argument('kwarg3', types.NumberMultichoice, choices=(1, 2, 3, 4, 5)) + def complex_func(arg1, arg2, kwarg1=('one',), kwarg2=4, kwarg3=(1, 2)): + return { + 'arg1': arg1, + 'arg2': arg2, + 'kwarg1': kwarg1, + 'kwarg2': kwarg2, + 'kwarg3': kwarg3 + } + desc = complex_func.describe() assert len(desc['inputs']) == 5 assert desc['name'].split('.')[-1] == 'complex_func' @@ -129,7 +134,14 @@ def test_multiple_arguments(): } +@deprecation.fail_if_not_removed def test_girder_input_mode(): + + @argument('item', types.GirderItem) + @argument('folder', types.GirderFolder) + def girder_types_func(item, folder): + return item, folder + item, folder = girder_types_func.call_item_task({ 'item': { 'mode': 'girder', @@ -148,6 +160,7 @@ def test_girder_input_mode(): assert folder == 'folderid' +@deprecation.fail_if_not_removed def test_missing_description_exception(): def func(): pass @@ -156,11 +169,13 @@ def func(): decorators.get_description_attribute(func) +@deprecation.fail_if_not_removed def test_argument_name_not_string(): with pytest.raises(TypeError): argument(0, types.Integer) +@deprecation.fail_if_not_removed def test_argument_name_not_a_parameter(): with pytest.raises(ValueError): @argument('notarg', types.Integer) @@ -168,7 +183,153 @@ def func(arg): pass +@deprecation.fail_if_not_removed def test_unhandled_input_binding(): arg = argument('arg', types.Integer) with pytest.raises(ValueError): decorators.get_input_data(arg, {}) + +########################### + +def arg(a): pass # noqa +def varargs(*args): pass # noqa +def kwarg(a='test'): pass # noqa +def kwargs(**kwargs): pass # noqa +def arg_arg(a, b): pass # noqa +def arg_varargs(a, *args): pass # noqa +def arg_kwarg(a, b='test'): pass # noqa +def arg_kwargs(a, **kwargs): pass # noqa +def kwarg_varargs(a='test', *args): pass # noqa +def kwarg_kwarg(a='testa', b='testb'): pass # noqa +def kwarg_kwargs(a='test', **kwargs): pass # noqa +def arg_kwarg_varargs(a, b='test', *args): pass # noqa +def arg_kwarg_kwargs(a, b='test', **kwargs): pass # noqa +def arg_kwarg_varargs_kwargs(a, b='test', *args, **kwargs): pass # noqa + + +@pytest.mark.parametrize('func,classes', [ + (arg, [PositionalArg]), + (varargs, [VarsArg]), + (kwarg, [KeywordArg]), + (kwargs, [KwargsArg]), + (arg_arg, [PositionalArg, PositionalArg]), + (arg_varargs, [PositionalArg, VarsArg]), + (arg_kwarg, [PositionalArg, KeywordArg]), + (arg_kwargs, [PositionalArg, KwargsArg]), + (kwarg_varargs, [KeywordArg, VarsArg]), + (kwarg_kwarg, [KeywordArg, KeywordArg]), + (kwarg_kwargs, [KeywordArg, KwargsArg]), + (arg_kwarg_varargs, [PositionalArg, KeywordArg, VarsArg]), + (arg_kwarg_kwargs, [PositionalArg, KeywordArg, KwargsArg]), + (arg_kwarg_varargs_kwargs, [PositionalArg, KeywordArg, VarsArg, KwargsArg]) +]) +def test_GWFuncDesc_arguments_returns_expected_classes(func, classes): + spec = GWFuncDesc(func) + assert len(spec.arguments) == len(classes) + for arg, cls in zip(spec.arguments, classes): + assert isinstance(arg, cls) + + +no_varargs = [arg, kwarg, kwargs, arg_arg, arg_kwarg, + arg_kwargs, kwarg_kwarg, kwarg_kwargs, + arg_kwarg_kwargs] + + +@pytest.mark.parametrize('func', no_varargs) +def test_GWFuncDesc_varargs_returns_None(func): + spec = GWFuncDesc(func) + assert spec.varargs is None + + +with_varargs = [varargs, arg_varargs, kwarg_varargs, + arg_kwarg_varargs, arg_kwarg_varargs_kwargs] + + +@pytest.mark.parametrize('func', with_varargs) +def test_GWFuncDesc_varargs_returns_Vararg(func): + spec = GWFuncDesc(func) + assert isinstance(spec.varargs, VarsArg) + + +@pytest.mark.parametrize('func,names', [ + (arg, ["a"]), + (arg_arg, ["a", "b"]), + (arg_varargs, ["a"]), + (arg_kwarg, ["a"]), + (arg_kwargs, ["a"]), + (arg_kwarg_kwargs, ["a"]), + (arg_kwarg_varargs_kwargs, ["a"]) +]) +def test_GWFuncDesc_positional_args_correct_names(func, names): + spec = GWFuncDesc(func) + assert len(spec.positional_args) == len(names) + for p, n in zip(spec.positional_args, names): + assert isinstance(p, PositionalArg) + assert p.name == n + + +@pytest.mark.parametrize('func', [ + varargs, kwarg, kwargs, kwarg_varargs, + kwarg_kwarg, kwarg_kwargs +]) +def test_GWFuncDesc_positional_args_returns_empty_list_if_no_positional_args(func): + spec = GWFuncDesc(func) + assert len(spec.positional_args) == 0 + + +@pytest.mark.parametrize('func,names', [ + (kwarg, ['a']), + (arg_kwarg, ['b']), + (kwarg_varargs, ['a']), + (kwarg_kwarg, ['a', 'b']), + (kwarg_kwargs, ['a']), + (arg_kwarg_varargs, ['b']), + (arg_kwarg_kwargs, ['b']), + (arg_kwarg_varargs_kwargs, ['b']), +]) +def test_GWFuncDesc_keyword_args_correct_names(func, names): + spec = GWFuncDesc(func) + assert len(spec.keyword_args) == len(names) + for p, n in zip(spec.keyword_args, names): + assert isinstance(p, KeywordArg) + assert p.name == n + + +@pytest.mark.parametrize('func', [ + arg, varargs, kwargs, arg_arg, arg_varargs, arg_kwargs +]) +def test_GWFuncDesc_keyword_args_returns_empty_list_if_no_keyword_args(func): + spec = GWFuncDesc(func) + assert len(spec.keyword_args) == 0 + + +@pytest.mark.parametrize('func,defaults', [ + (kwarg, ['test']), + (arg_kwarg, ['test']), + (kwarg_varargs, ['test']), + (kwarg_kwarg, ['testa', 'testb']), + (kwarg_kwargs, ['test']), + (arg_kwarg_varargs, ['test']), + (arg_kwarg_kwargs, ['test']), + (arg_kwarg_varargs_kwargs, ['test']), +]) +def test_GWFuncDesc_keyword_args_have_defaults(func, defaults): + spec = GWFuncDesc(func) + assert len(spec.keyword_args) == len(defaults) + for p, d in zip(spec.keyword_args, defaults): + assert hasattr(p, 'default') + assert p.default == d + + +def test_parameter_decorator_adds_metadata(): + @parameter('a', test='TEST') + def arg(a): + pass + + desc = GWFuncDesc.get_description(arg) + + assert hasattr(desc.arguments[0], 'test') + assert desc.arguments[0].test == 'TEST' + + +# TODO more tests around other argument types (e.g. kwargs varargs etc) diff --git a/girder_worker_utils/tests/py3_decorators_test.py b/girder_worker_utils/tests/py3_decorators_test.py new file mode 100644 index 0000000..d5d59d8 --- /dev/null +++ b/girder_worker_utils/tests/py3_decorators_test.py @@ -0,0 +1,109 @@ +import pytest + +from girder_worker_utils.decorators import ( + GWFuncDesc, + KeywordArg, + PositionalArg, + VarsArg) + + +def arg_varargs_kwarg(a, *args, b='test'): pass # noqa +def arg_varargs_kwarg_no_default(a, *args, b): pass # noqa +def arg_emptyvarargs_kwarg(a, *, b='test'): pass # noqa +def arg_emptyvarargs_kwarg_no_default(a, *, b): pass # noqa +def arg_with_annotation(a: int): pass # noqa + + +@pytest.mark.parametrize('func,classes', [ + (arg_varargs_kwarg, [PositionalArg, VarsArg, KeywordArg]), + (arg_varargs_kwarg_no_default, [PositionalArg, VarsArg, KeywordArg]), + (arg_emptyvarargs_kwarg, [PositionalArg, KeywordArg]), + (arg_emptyvarargs_kwarg_no_default, [PositionalArg, KeywordArg]) +]) +def test_GWFuncDesc_arguments_returns_expected_classes(func, classes): + spec = GWFuncDesc(func) + assert len(spec.arguments) == len(classes) + for arg, cls in zip(spec.arguments, classes): + assert isinstance(arg, cls) + + +no_varargs = [arg_emptyvarargs_kwarg, + arg_emptyvarargs_kwarg_no_default] + + +@pytest.mark.parametrize('func', no_varargs) +def test_GWFuncDesc_varargs_returns_None(func): + spec = GWFuncDesc(func) + assert spec.varargs is None + + +with_varargs = [ + arg_varargs_kwarg, + arg_varargs_kwarg_no_default] + + +@pytest.mark.parametrize('func', with_varargs) +def test_GWFuncDesc_varargs_returns_Vararg(func): + spec = GWFuncDesc(func) + assert isinstance(spec.varargs, VarsArg) + + +@pytest.mark.parametrize('func,names', [ + (arg_varargs_kwarg, ["a"]), + (arg_varargs_kwarg_no_default, ["a"]), + (arg_emptyvarargs_kwarg, ["a"]), + (arg_emptyvarargs_kwarg_no_default, ["a"]), +]) +def test_GWFuncDesc_positional_args_correct_names(func, names): + spec = GWFuncDesc(func) + assert len(spec.positional_args) == len(names) + for p, n in zip(spec.positional_args, names): + assert isinstance(p, PositionalArg) + assert p.name == n + + +@pytest.mark.parametrize('func,names', [ + (arg_varargs_kwarg, ['b']), + (arg_varargs_kwarg_no_default, ['b']), + (arg_emptyvarargs_kwarg, ['b']), + (arg_emptyvarargs_kwarg_no_default, ['b']) +]) +def test_GWFuncDesc_keyword_args_correct_names(func, names): + spec = GWFuncDesc(func) + assert len(spec.keyword_args) == len(names) + for p, n in zip(spec.keyword_args, names): + assert isinstance(p, KeywordArg) + assert p.name == n + + +@pytest.mark.parametrize('func,defaults', [ + (arg_varargs_kwarg, ['test']), + (arg_emptyvarargs_kwarg, ['test']) +]) +def test_GWFuncDesc_keyword_args_have_defaults(func, defaults): + spec = GWFuncDesc(func) + assert len(spec.keyword_args) == len(defaults) + for p, d in zip(spec.keyword_args, defaults): + assert hasattr(p, 'default') + assert p.default == d + + +@pytest.mark.parametrize('func', [ + arg_varargs_kwarg_no_default, + arg_emptyvarargs_kwarg_no_default +]) +def test_GWFuncDesc_keyword_args_with_no_defaults_have_no_defaults(func): + spec = GWFuncDesc(func) + for p in spec.keyword_args: + assert not hasattr(p, "default") + + +@pytest.mark.parametrize('func,data_types', [ + (arg_with_annotation, [int]) +]) +def test_GWFuncDesc_positional_args_with_annotation_have_data_type(func, data_types): + spec = GWFuncDesc(func) + assert len(spec.positional_args) == len(data_types) + for p, d in zip(spec.positional_args, data_types): + assert hasattr(p, "data_type") + assert p.data_type == d diff --git a/requirements.in b/requirements.in index 23a8c80..223976a 100644 --- a/requirements.in +++ b/requirements.in @@ -3,3 +3,4 @@ girder-client>=2 jsonpickle setuptools_scm six +deprecation diff --git a/requirements.txt b/requirements.txt index 8dce4b5..720a3b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,26 +8,30 @@ certifi==2017.7.27.1 # via requests chardet==3.0.4 # via requests click==6.7 # via girder-client coverage==4.4.1 # via pytest-cov +deprecation==2.0.6 diskcache==2.9.0 # via girder-client flake8-blind-except==0.1.1 flake8-docstrings==1.1.0 flake8-import-order==0.13 flake8-polyfill==1.0.1 # via flake8-docstrings flake8==3.4.1 -funcsigs==1.0.2 ; python_version < "3.5" girder-client==2.3.0 idna==2.6 # via requests +jsonpickle==1.0 mccabe==0.6.1 # via flake8 mock==2.0.0 +packaging==18.0 # via deprecation pbr==3.1.1 # via mock py==1.4.34 # via pytest pycodestyle==2.3.1 # via flake8, flake8-import-order pydocstyle==2.0.0 # via flake8-docstrings pyflakes==1.5.0 # via flake8 +pyparsing==2.2.2 # via packaging pytest-cov==2.5.1 pytest==3.2.3 requests-toolbelt==0.8.0 # via girder-client requests==2.18.4 # via girder-client, requests-toolbelt +setuptools-scm==3.1.0 six==1.11.0 snowballstemmer==1.2.1 # via pydocstyle urllib3==1.22 # via requests diff --git a/setup.py b/setup.py index 6688897..b8ff2b2 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ def prerelease_local_scheme(version): author_email='kitware@kitware.com', license='Apache 2.0', classifiers=[ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 3 - Alpha', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python',