Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Don’t directly import numpy and add gen_types decorator #966

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions param/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -612,3 +612,15 @@ def async_executor(func):
task.add_done_callback(_running_tasks.discard)
else:
event_loop.run_until_complete(func())


def anyinstance(obj, class_tuple_generator):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could an approach like this be used to avoid creating these functions? https://peps.python.org/pep-3119/#overloading-isinstance-and-issubclass

Copy link
Member Author

@hoxbro hoxbro Sep 30, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think so, though I haven't read the PEP in detail.

However, it seems pretty advanced to use for a simple conversion.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It seems it'd be something like:

import sys

class GeneratorIsMeta(type):
    def __instancecheck__(cls, inst):
        return isinstance(inst, tuple(cls.gen_types()))

    def __subclasscheck__(cls, sub):
        return issubclass(sub, tuple(cls.gen_types()))


class DtTypes(metaclass=GeneratorIsMeta):
    @classmethod
    def gen_types(cls):
        yield dt.datetime
        yield dt.date
        if "numpy" in sys.modules:
            import numpy as np
            yield np.datetime64

class IntTypes(metaclass=GeneratorIsMeta):
    @classmethod
    def gen_types(cls):
        yield int
        if "numpy" in sys.modules:
            import numpy as np
            yield np.integer

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you.

I still thinks it is pretty advanced (even though you don't need it to be iterable in your example).

You could likely move the advanced logic into a decorator, but then you would need to import that into our other libraries. My main point with accepting iterator is not so much for param itself but for HoloViews.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My main point with accepting iterator is not so much for param itself but for HoloViews.

Oh yeah I agree users shouldn't have to pass this custom class, a generator seems appropriate. Instead I was wondering if there couldn't be a way to use the metaclass approach internally to avoid the special anyisinstance and anyissubclass functions, they seem to be easy to forget in the future.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have switched to a metaclass approach with a decorator for easy access.

if inspect.isgeneratorfunction(class_tuple_generator):
class_tuple_generator = tuple(class_tuple_generator())
return isinstance(obj, class_tuple_generator)


def anysubclass(obj, class_tuple_generator):
if inspect.isgeneratorfunction(class_tuple_generator):
class_tuple_generator = tuple(class_tuple_generator())
return issubclass(obj, class_tuple_generator)
52 changes: 32 additions & 20 deletions param/parameterized.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
import logging
import numbers
import operator
import random
import re
import sys
import types
import typing
import warnings
Expand Down Expand Up @@ -74,15 +74,18 @@

from inspect import getfullargspec

dt_types = (dt.datetime, dt.date)
_int_types = (int,)
def _dt_types():
yield dt.datetime
yield dt.date
if "numpy" in sys.modules:
import numpy as np
yield np.datetime64

try:
import numpy as np
dt_types = dt_types + (np.datetime64,)
_int_types = _int_types + (np.integer,)
except:
pass
def _int_types():
yield int
if "numpy" in sys.modules:
import numpy as np
yield np.integer

VERBOSE = INFO - 1
logging.addLevelName(VERBOSE, "VERBOSE")
Expand Down Expand Up @@ -1714,11 +1717,18 @@ class Comparator:
type(None): operator.eq,
lambda o: hasattr(o, '_infinitely_iterable'): operator.eq, # Time
}
equalities.update({dtt: operator.eq for dtt in dt_types})
gen_equalities = {
_dt_types: operator.eq
}

@classmethod
def is_equal(cls, obj1, obj2):
for eq_type, eq in cls.equalities.items():
equals = cls.equalities.copy()
for gen, op in cls.gen_equalities.items():
for t in gen():
equals[t] = op

for eq_type, eq in equals.items():
try:
are_instances = isinstance(obj1, eq_type) and isinstance(obj2, eq_type)
except TypeError:
Expand Down Expand Up @@ -3804,6 +3814,9 @@ def pprint(val,imports=None, prefix="\n ", settings=[],
elif type(val) in script_repr_reg:
rep = script_repr_reg[type(val)](val,imports,prefix,settings)

elif type(val) in (_np_random(), _py_random()):
rep = None

elif isinstance(val, Parameterized) or (type(val) is type and issubclass(val, Parameterized)):
rep=val.param.pprint(imports=imports, prefix=prefix+" ",
qualify=qualify, unknown_value=unknown_value,
Expand Down Expand Up @@ -3838,17 +3851,16 @@ def container_script_repr(container,imports,prefix,settings):
return rep


def empty_script_repr(*args): # pyflakes:ignore (unused arguments):
return None
def _np_random():
if "numpy" in sys.modules:
hoxbro marked this conversation as resolved.
Show resolved Hide resolved
import numpy as np
return np.random.RandomState

try:
# Suppress scriptrepr for objects not yet having a useful string representation
import numpy
script_repr_reg[random.Random] = empty_script_repr
script_repr_reg[numpy.random.RandomState] = empty_script_repr

except ImportError:
pass # Support added only if those libraries are available
def _py_random():
if "random" in sys.modules:
import random
return random.Random


def function_script_repr(fn,imports,prefix,settings):
Expand Down
42 changes: 23 additions & 19 deletions param/parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

from .parameterized import (
Parameterized, Parameter, ParameterizedFunction, ParamOverrides, String,
Undefined, get_logger, instance_descriptor, dt_types,
Undefined, get_logger, instance_descriptor, _dt_types,
_int_types, _identity_hook
)
from ._utils import (
Expand All @@ -51,6 +51,8 @@
concrete_descendents,
_abbreviate_paths,
_to_datetime,
anyinstance,
anysubclass,
)

#-----------------------------------------------------------------------------
Expand Down Expand Up @@ -94,7 +96,7 @@ def guess_param_types(**kwargs):
kws = dict(default=v, constant=True)
if isinstance(v, Parameter):
params[k] = v
elif isinstance(v, dt_types):
elif anyinstance(v, _dt_types):
params[k] = Date(**kws)
elif isinstance(v, bool):
params[k] = Boolean(**kws)
Expand All @@ -109,7 +111,7 @@ def guess_param_types(**kwargs):
elif isinstance(v, tuple):
if all(_is_number(el) for el in v):
params[k] = NumericTuple(**kws)
elif all(isinstance(el, dt_types) for el in v) and len(v)==2:
elif all(anyinstance(el, _dt_types) for el in v) and len(v)==2:
hoxbro marked this conversation as resolved.
Show resolved Hide resolved
params[k] = DateRange(**kws)
else:
params[k] = Tuple(**kws)
Expand Down Expand Up @@ -141,7 +143,7 @@ def parameterized_class(name, params, bases=Parameterized):
Dynamically create a parameterized class with the given name and the
supplied parameters, inheriting from the specified base(s).
"""
if not (isinstance(bases, list) or isinstance(bases, tuple)):
if not isinstance(bases, (list, tuple)):
bases=[bases]
return type(name, tuple(bases), params)

Expand Down Expand Up @@ -852,7 +854,7 @@ def _validate_value(self, val, allow_None):
if allow_None and val is None:
return

if not isinstance(val, _int_types):
if not anyinstance(val, _int_types):
raise ValueError(
f"{_validate_error_prefix(self)} must be an integer, "
f"not {type(val)}."
Expand Down Expand Up @@ -917,14 +919,14 @@ def _validate_value(self, val, allow_None):
if self.allow_None and val is None:
return

if not isinstance(val, dt_types) and not (allow_None and val is None):
if not anyinstance(val, _dt_types) and not (allow_None and val is None):
raise ValueError(
f"{_validate_error_prefix(self)} only takes datetime and "
f"date types, not {type(val)}."
)

def _validate_step(self, val, step):
if step is not None and not isinstance(step, dt_types):
if step is not None and not anyinstance(step, _dt_types):
raise ValueError(
f"{_validate_error_prefix(self, 'step')} can only be None, "
f"a datetime or date type, not {type(step)}."
Expand Down Expand Up @@ -1355,7 +1357,7 @@ class DateRange(Range):
"""

def _validate_bound_type(self, value, position, kind):
if not isinstance(value, dt_types):
if not anyinstance(value, _dt_types):
raise ValueError(
f"{_validate_error_prefix(self)} {position} {kind} can only be "
f"None or a date/datetime value, not {type(value)}."
Expand All @@ -1379,7 +1381,7 @@ def _validate_value(self, val, allow_None):
f"not {type(val)}."
)
for n in val:
if isinstance(n, dt_types):
if anyinstance(n, _dt_types):
continue
raise ValueError(
f"{_validate_error_prefix(self)} only takes date/datetime "
Expand Down Expand Up @@ -2184,18 +2186,20 @@ def _validate(self, val):
def _validate_class_(self, val, class_, is_instance):
if (val is None and self.allow_None):
return
if (is_instance and anyinstance(val, class_)) or (not is_instance and anysubclass(val, class_)):
return

if isinstance(class_, tuple):
class_name = ('(%s)' % ', '.join(cl.__name__ for cl in class_))
class_name = ('({})'.format(', '.join(cl.__name__ for cl in class_)))
elif inspect.isgenerator(class_):
hoxbro marked this conversation as resolved.
Show resolved Hide resolved
class_name = ('({})'.format(', '.join(cl.__name__ for cl in class_())))
else:
class_name = class_.__name__
if is_instance:
if not (isinstance(val, class_)):
raise ValueError(
f"{_validate_error_prefix(self)} value must be an instance of {class_name}, not {val!r}.")
else:
if not (issubclass(val, class_)):
raise ValueError(
f"{_validate_error_prefix(self)} value must be a subclass of {class_name}, not {val}.")

raise ValueError(
f"{_validate_error_prefix(self)} value must be "
f"{'an instance' if is_instance else 'a subclass'} of {class_name}, not {val!r}."
)

def get_range(self):
"""
Expand Down Expand Up @@ -2559,7 +2563,7 @@ def _validate_item_type(self, val, item_type):
if item_type is None or (self.allow_None and val is None):
return
for v in val:
if isinstance(v, item_type):
if anyinstance(v, item_type):
continue
raise TypeError(
f"{_validate_error_prefix(self)} items must be instances "
Expand Down
20 changes: 20 additions & 0 deletions tests/testimports.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import sys
from subprocess import check_output
from textwrap import dedent


def test_no_blocklist_imports():
check = """\
import sys
import param
blocklist = {"numpy", "IPython", "pandas"}
mods = blocklist & set(sys.modules)
if mods:
print(", ".join(mods), end="")
"""

output = check_output([sys.executable, '-c', dedent(check)])

assert output == b""
Loading