From f6dd31bf3529d7f8886a0c3f08510981fd0e8f9a Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 25 Nov 2021 17:55:08 -0500 Subject: [PATCH 01/56] start checking mypy and work on some hints (cherry picked from commit 60e74333b5525143200a368a549a3c37127d8af6) --- .github/workflows/main.yml | 2 ++ clvm/CLVMObject.py | 11 ++++++++++- clvm/SExp.py | 15 ++++++++------- clvm/more_ops.py | 2 ++ clvm/operators.py | 18 ++++++++++++++---- clvm/run_program.py | 15 +++++++++------ mypy.ini | 5 +++++ setup.py | 3 +++ 8 files changed, 53 insertions(+), 18 deletions(-) create mode 100644 mypy.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f5b1bab3..3af4c2c0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,6 +29,8 @@ jobs: python -m pip install flake8 - name: flake8 run: flake8 clvm tests --max-line-length=120 + - name: mypy + run: mypy - name: Test with pytest run: | py.test tests diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 7586968c..6445e444 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -1,5 +1,14 @@ import typing +from typing_extensions import Protocol + + +class CLVMObjectLike(Protocol): + # It's not clear if it is possible to express the exclusivity without maybe + # restructuring all the classes. + atom: typing.Optional[bytes] + pair: typing.Optional[typing.Tuple["CLVMObjectLike", "CLVMObjectLike"]] + class CLVMObject: """ @@ -11,7 +20,7 @@ class CLVMObject: # this is always a 2-tuple of an object implementing the CLVM object # protocol. - pair: typing.Optional[typing.Tuple[typing.Any, typing.Any]] + pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] __slots__ = ["atom", "pair"] def __new__(class_, v): diff --git a/clvm/SExp.py b/clvm/SExp.py index 398fe108..d54801b1 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -4,7 +4,7 @@ from blspy import G1Element from .as_python import as_python -from .CLVMObject import CLVMObject +from .CLVMObject import CLVMObject, CLVMObjectLike from .EvalError import EvalError @@ -24,7 +24,7 @@ None, G1Element, list, - typing.Tuple[typing.Any, typing.Any], + typing.Tuple["CastableType", "CastableType"], ] @@ -60,13 +60,14 @@ def convert_atom_to_bytes( # returns a clvm-object like object def to_sexp_type( v: CastableType, -): +) -> CLVMObjectLike: stack = [v] - ops = [(0, None)] # convert + ops: typing.List[typing.Tuple[int, typing.Optional[int]]] = [(0, None)] # convert while len(ops) > 0: op, target = ops.pop() - # convert value + # convert valuefrom .operators import OperatorDict + if op == 0: if looks_like_clvm_object(stack[-1]): continue @@ -140,14 +141,14 @@ class SExp: # this is a tuple of the otherlying CLVMObject-like objects. i.e. not # SExp objects with higher level functions, or None - pair: typing.Optional[typing.Tuple[typing.Any, typing.Any]] + pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] def __init__(self, obj): self.atom = obj.atom self.pair = obj.pair # this returns a tuple of two SExp objects, or None - def as_pair(self) -> typing.Tuple["SExp", "SExp"]: + def as_pair(self) -> typing.Optional[typing.Tuple["SExp", "SExp"]]: pair = self.pair if pair is None: return pair diff --git a/clvm/more_ops.py b/clvm/more_ops.py index 43df6cb5..0bd5df3b 100644 --- a/clvm/more_ops.py +++ b/clvm/more_ops.py @@ -51,6 +51,8 @@ def malloc_cost(cost, atom: SExp): + if atom.atom is None: + raise ValueError("Atom must have a non-None atom attribute") return cost + len(atom.atom) * MALLOC_COST_PER_BYTE, atom diff --git a/clvm/operators.py b/clvm/operators.py index a63e6a88..ebb7e458 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,5 +1,7 @@ from typing import Dict, Tuple +from typing_extensions import Protocol + from . import core_ops, more_ops from .CLVMObject import CLVMObject @@ -65,7 +67,7 @@ } -def args_len(op_name, args): +def args_len(op_name, args: SExp): for arg in args.as_iter(): if arg.pair: raise EvalError("%s requires int args" % op_name, arg) @@ -99,7 +101,7 @@ def args_len(op_name, args): # this means that unknown ops where cost_function is 1, 2, or 3, may still be # fatal errors if the arguments passed are not atoms. -def default_unknown_op(op: bytes, args: CLVMObject) -> Tuple[int, CLVMObject]: +def default_unknown_op(op: bytes, args: SExp) -> Tuple[int, SExp]: # any opcode starting with ffff is reserved (i.e. fatal error) # opcodes are not allowed to be empty if len(op) == 0 or op[:2] == b"\xff\xff": @@ -165,13 +167,21 @@ def default_unknown_op(op: bytes, args: CLVMObject) -> Tuple[int, CLVMObject]: return (cost, SExp.null()) +class DefaultOperator(Protocol): + def __call__(self, op: bytes, args: SExp) -> Tuple[int, SExp]: ... + + class OperatorDict(dict): """ This is a nice hack that adds `__call__` to a dictionary, so operators can be added dynamically. """ - def __new__(class_, d: Dict, *args, **kwargs): + unknown_op_handler: DefaultOperator + quote_atom: int + apply_atom: int + + def __new__(class_, d: "OperatorDict", *args, **kwargs): """ `quote_atom` and `apply_atom` must be set `unknown_op_handler` has a default implementation @@ -187,7 +197,7 @@ def __new__(class_, d: Dict, *args, **kwargs): self.unknown_op_handler = default_unknown_op return self - def __call__(self, op: bytes, arguments: CLVMObject) -> Tuple[int, CLVMObject]: + def __call__(self, op: bytes, arguments: SExp) -> Tuple[int, SExp]: f = self.get(op) if f is None: return self.unknown_op_handler(op, arguments) diff --git a/clvm/run_program.py b/clvm/run_program.py index 20f4b75c..f8f36471 100644 --- a/clvm/run_program.py +++ b/clvm/run_program.py @@ -3,6 +3,7 @@ from .CLVMObject import CLVMObject from .EvalError import EvalError from .SExp import SExp +from .operators import OperatorDict from .costs import ( APPLY_COST, @@ -48,14 +49,14 @@ def msb_mask(byte): def run_program( program: CLVMObject, args: CLVMObject, - operator_lookup: Callable[[bytes, CLVMObject], Tuple[int, CLVMObject]], + operator_lookup: OperatorDict, max_cost=None, pre_eval_f=None, -) -> Tuple[int, CLVMObject]: +) -> Tuple[int, SExp]: - program = SExp.to(program) + _program = SExp.to(program) if pre_eval_f: - pre_eval_op = to_pre_eval_op(pre_eval_f, program.to) + pre_eval_op = to_pre_eval_op(pre_eval_f, _program.to) else: pre_eval_op = None @@ -65,6 +66,8 @@ def traverse_path(sexp: SExp, env: SExp) -> Tuple[int, SExp]: if sexp.nullp(): return cost, sexp.null() + if sexp.atom is None: + raise ValueError("Atom must have a non-None atom attribute") b = sexp.atom end_byte_cursor = 0 @@ -174,12 +177,12 @@ def apply_op(op_stack: OpStackType, value_stack: ValStackType) -> int: return additional_cost op_stack: OpStackType = [eval_op] - value_stack: ValStackType = [program.cons(args)] + value_stack: ValStackType = [_program.cons(args)] cost: int = 0 while op_stack: f = op_stack.pop() cost += f(op_stack, value_stack) if max_cost and cost > max_cost: - raise EvalError("cost exceeded", program.to(max_cost)) + raise EvalError("cost exceeded", _program.to(max_cost)) return cost, value_stack[-1] diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..39345d44 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,5 @@ +[mypy] +files = clvm,tests,*.py +;ignore_missing_imports = True +show_error_codes = True +warn_unused_ignores = True diff --git a/setup.py b/setup.py index a1a86be7..6754b2fd 100755 --- a/setup.py +++ b/setup.py @@ -7,11 +7,14 @@ dependencies = [ "blspy>=0.9", + "typing-extensions~=4.0.0", # Backports of new typing module features ] dev_dependencies = [ "clvm_tools>=0.4.2", + "mypy", "pytest", + "types-setuptools", ] setup( From ac4d446438b3fb385836b9dd9ce9ab9ab1366674 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 25 Nov 2021 17:58:48 -0500 Subject: [PATCH 02/56] remove unused imports (cherry picked from commit 9741f9c3d42c8606df9f99b534c9d28024d250e8) --- clvm/operators.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clvm/operators.py b/clvm/operators.py index ebb7e458..68686021 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,10 +1,9 @@ -from typing import Dict, Tuple +from typing import Tuple from typing_extensions import Protocol from . import core_ops, more_ops -from .CLVMObject import CLVMObject from .SExp import SExp from .EvalError import EvalError From 6706e20f89fbd0e815d57c0998a7289d2978a00f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 26 Nov 2021 13:38:52 -0500 Subject: [PATCH 03/56] some more mypy (cherry picked from commit 65b0fb88364e5207a8f75513c14bad4589870b70) --- clvm/CLVMObject.py | 16 +++++++++-- clvm/SExp.py | 36 +++++++++++++----------- clvm/casts.py | 6 ++-- clvm/more_ops.py | 68 +++++++++++++++++++++++++++++----------------- clvm/serialize.py | 40 +++++++++++++++------------ mypy.ini | 3 ++ 6 files changed, 105 insertions(+), 64 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 6445e444..81594abe 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -10,6 +10,9 @@ class CLVMObjectLike(Protocol): pair: typing.Optional[typing.Tuple["CLVMObjectLike", "CLVMObjectLike"]] +_T_CLVMObject = typing.TypeVar("_T_CLVMObject") + + class CLVMObject: """ This class implements the CLVM Object protocol in the simplest possible way, @@ -23,13 +26,20 @@ class CLVMObject: pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] __slots__ = ["atom", "pair"] - def __new__(class_, v): - if isinstance(v, CLVMObject): + @staticmethod + def __new__( + class_: typing.Type[_T_CLVMObject], + v: typing.Union["CLVMObject", typing.Tuple, bytes], + ) -> _T_CLVMObject: + if isinstance(v, class_): return v self = super(CLVMObject, class_).__new__(class_) if isinstance(v, tuple): if len(v) != 2: - raise ValueError("tuples must be of size 2, cannot create CLVMObject from: %s" % str(v)) + raise ValueError( + "tuples must be of size 2, cannot create CLVMObject from: %s" + % str(v) + ) self.pair = v self.atom = None else: diff --git a/clvm/SExp.py b/clvm/SExp.py index d54801b1..f5463334 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -118,6 +118,7 @@ def to_sexp_type( return stack[0] +# TODO: Maybe there is some way to track atom vs. pair SExps to help hinting out a bit class SExp: """ SExp provides higher level API on top of any object implementing the CLVM @@ -143,7 +144,7 @@ class SExp: # SExp objects with higher level functions, or None pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] - def __init__(self, obj): + def __init__(self, obj: CLVMObjectLike) -> None: self.atom = obj.atom self.pair = obj.pair @@ -155,20 +156,22 @@ def as_pair(self) -> typing.Optional[typing.Tuple["SExp", "SExp"]]: return (self.__class__(pair[0]), self.__class__(pair[1])) # TODO: deprecate this. Same as .atom property - def as_atom(self): + def as_atom(self) -> typing.Optional[bytes]: return self.atom - def listp(self): + def listp(self) -> bool: return self.pair is not None - def nullp(self): + def nullp(self) -> bool: v = self.atom return v is not None and len(v) == 0 - def as_int(self): + def as_int(self) -> int: + if self.atom is None: + raise TypeError("Unable to convert a pair to an int") return int_from_bytes(self.atom) - def as_bin(self): + def as_bin(self) -> bytes: f = io.BytesIO() sexp_to_stream(self, f) return f.getvalue() @@ -179,37 +182,38 @@ def to(class_, v: CastableType) -> "SExp": return v if looks_like_clvm_object(v): - return class_(v) + # TODO: maybe this can be done more cleanly + return class_(typing.cast(CLVMObjectLike, v)) # this will lazily convert elements return class_(to_sexp_type(v)) - def cons(self, right): + def cons(self: _T_SExp, right) -> _T_SExp: return self.to((self, right)) - def first(self): + def first(self: _T_SExp) -> _T_SExp: pair = self.pair if pair: return self.__class__(pair[0]) raise EvalError("first of non-cons", self) - def rest(self): + def rest(self: _T_SExp) -> _T_SExp: pair = self.pair if pair: return self.__class__(pair[1]) raise EvalError("rest of non-cons", self) @classmethod - def null(class_): + def null(class_) -> "SExp": return class_.__null__ - def as_iter(self): + def as_iter(self: _T_SExp) -> typing.Iterable[_T_SExp]: v = self while not v.nullp(): yield v.first() v = v.rest() - def __eq__(self, other: CastableType): + def __eq__(self, other: CastableType) -> bool: try: other = self.to(other) to_compare_stack = [(self, other)] @@ -229,7 +233,7 @@ def __eq__(self, other: CastableType): except ValueError: return False - def list_len(self): + def list_len(self) -> int: v = self size = 0 while v.listp(): @@ -240,10 +244,10 @@ def list_len(self): def as_python(self): return as_python(self) - def __str__(self): + def __str__(self) -> str: return self.as_bin().hex() - def __repr__(self): + def __repr__(self) -> str: return "%s(%s)" % (self.__class__.__name__, str(self)) diff --git a/clvm/casts.py b/clvm/casts.py index d142c3a4..1a49886e 100644 --- a/clvm/casts.py +++ b/clvm/casts.py @@ -1,11 +1,11 @@ -def int_from_bytes(blob): +def int_from_bytes(blob: bytes) -> int: size = len(blob) if size == 0: return 0 return int.from_bytes(blob, "big", signed=True) -def int_to_bytes(v): +def int_to_bytes(v: int) -> bytes: byte_count = (v.bit_length() + 8) >> 3 if v == 0: return b"" @@ -17,7 +17,7 @@ def int_to_bytes(v): return r -def limbs_for_int(v): +def limbs_for_int(v: int) -> int: """ Return the number of bytes required to represent this integer. """ diff --git a/clvm/more_ops.py b/clvm/more_ops.py index 0bd5df3b..2f92b070 100644 --- a/clvm/more_ops.py +++ b/clvm/more_ops.py @@ -1,5 +1,6 @@ import hashlib import io +import typing from blspy import G1Element, PrivateKey @@ -50,13 +51,16 @@ ) -def malloc_cost(cost, atom: SExp): +_T_SExp = typing.TypeVar("_T_SExp", bound=SExp) + + +def malloc_cost(cost: int, atom: _T_SExp) -> typing.Tuple[int, _T_SExp]: if atom.atom is None: raise ValueError("Atom must have a non-None atom attribute") return cost + len(atom.atom) * MALLOC_COST_PER_BYTE, atom -def op_sha256(args: SExp): +def op_sha256(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: cost = SHA256_BASE_COST arg_len = 0 h = hashlib.sha256() @@ -71,23 +75,24 @@ def op_sha256(args: SExp): return malloc_cost(cost, args.to(h.digest())) -def args_as_ints(op_name, args: SExp): +# TODO: can we get more specific about op_name such as with a typing.Literal? +def args_as_ints(op_name: str, args: SExp) -> typing.Iterator[typing.Tuple[int, int]]: for arg in args.as_iter(): - if arg.pair: + if arg.atom is None: raise EvalError("%s requires int args" % op_name, arg) - yield (arg.as_int(), len(arg.as_atom())) + yield (arg.as_int(), len(arg.atom)) -def args_as_int32(op_name, args: SExp): +def args_as_int32(op_name: str, args: SExp) -> typing.Iterator[int]: for arg in args.as_iter(): - if arg.pair: + if arg.atom is None: raise EvalError("%s requires int32 args" % op_name, arg) if len(arg.atom) > 4: raise EvalError("%s requires int32 args (with no leading zeros)" % op_name, arg) yield arg.as_int() -def args_as_int_list(op_name, args, count): +def args_as_int_list(op_name: str, args: SExp, count: int) -> typing.List[typing.Tuple[int, int]]: int_list = list(args_as_ints(op_name, args)) if len(int_list) != count: plural = "s" if count != 1 else "" @@ -95,7 +100,7 @@ def args_as_int_list(op_name, args, count): return int_list -def args_as_bools(op_name, args): +def args_as_bools(op_name: str, args: SExp) -> typing.Iterator[SExp]: for arg in args.as_iter(): v = arg.as_atom() if v == b"": @@ -104,7 +109,7 @@ def args_as_bools(op_name, args): yield args.true -def args_as_bool_list(op_name, args, count): +def args_as_bool_list(op_name: str, args: SExp, count: int) -> typing.List[SExp]: bool_list = list(args_as_bools(op_name, args)) if len(bool_list) != count: plural = "s" if count != 1 else "" @@ -112,7 +117,7 @@ def args_as_bool_list(op_name, args, count): return bool_list -def op_add(args: SExp): +def op_add(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: total = 0 cost = ARITH_BASE_COST arg_size = 0 @@ -124,7 +129,7 @@ def op_add(args: SExp): return malloc_cost(cost, args.to(total)) -def op_subtract(args): +def op_subtract(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: cost = ARITH_BASE_COST if args.nullp(): return malloc_cost(cost, args.to(0)) @@ -140,7 +145,7 @@ def op_subtract(args): return malloc_cost(cost, args.to(total)) -def op_multiply(args): +def op_multiply(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: cost = MUL_BASE_COST operands = args_as_ints("*", args) try: @@ -157,7 +162,7 @@ def op_multiply(args): return malloc_cost(cost, args.to(v)) -def op_divmod(args): +def op_divmod(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: cost = DIVMOD_BASE_COST (i0, l0), (i1, l1) = args_as_int_list("divmod", args, 2) if i1 == 0: @@ -166,11 +171,22 @@ def op_divmod(args): q, r = divmod(i0, i1) q1 = args.to(q) r1 = args.to(r) + if q1.atom is None: + # TODO: Should this (and other added TypeErrors) be EvalErrors? Using + # TypeErrors because that matches the type of the exception that would + # have been thrown in the existing code. This could also be done with + # something like below to avoid any runtime differences, but it is also + # a bit ugly. + # + # q1_atom: int = args.to(q).atom # type: ignore[assignment] + raise TypeError(f"Internal error, quotient must be an atom, got: {q1}") + if r1.atom is None: + raise TypeError(f"Internal error, quotient must be an atom, got: {r1}") cost += (len(q1.atom) + len(r1.atom)) * MALLOC_COST_PER_BYTE return cost, args.to((q, r)) -def op_div(args): +def op_div(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: cost = DIV_BASE_COST (i0, l0), (i1, l1) = args_as_int_list("/", args, 2) if i1 == 0: @@ -186,14 +202,14 @@ def op_div(args): return malloc_cost(cost, args.to(q)) -def op_gr(args): +def op_gr(args: SExp) -> typing.Tuple[int, SExp]: (i0, l0), (i1, l1) = args_as_int_list(">", args, 2) cost = GR_BASE_COST cost += (l0 + l1) * GR_COST_PER_BYTE return cost, args.true if i0 > i1 else args.false -def op_gr_bytes(args: SExp): +def op_gr_bytes(args: SExp) -> typing.Tuple[int, SExp]: arg_list = list(args.as_iter()) if len(arg_list) != 2: raise EvalError(">s takes exactly 2 arguments", args) @@ -203,11 +219,13 @@ def op_gr_bytes(args: SExp): b0 = a0.as_atom() b1 = a1.as_atom() cost = GRS_BASE_COST + if b0 is None or b1 is None: + raise TypeError(f"Internal error, both operands must not be None") cost += (len(b0) + len(b1)) * GRS_COST_PER_BYTE return cost, args.true if b0 > b1 else args.false -def op_pubkey_for_exp(args): +def op_pubkey_for_exp(args: _T_SExp) -> typing.Tuple[_T_SExp, _T_SExp]: ((i0, l0),) = args_as_int_list("pubkey_for_exp", args, 1) i0 %= 0x73EDA753299D7D483339D80809A1D80553BDA402FFFE5BFEFFFFFFFF00000001 exponent = PrivateKey.from_bytes(i0.to_bytes(32, "big")) @@ -220,7 +238,7 @@ def op_pubkey_for_exp(args): raise EvalError("problem in op_pubkey_for_exp: %s" % ex, args) -def op_point_add(items: SExp): +def op_point_add(items: _T_SExp) -> typing.Tuple[int, _T_SExp]: cost = POINT_ADD_BASE_COST p = G1Element() @@ -235,7 +253,7 @@ def op_point_add(items: SExp): return malloc_cost(cost, items.to(p)) -def op_strlen(args: SExp): +def op_strlen(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: if args.list_len() != 1: raise EvalError("strlen takes exactly 1 argument", args) a0 = args.first() @@ -246,7 +264,7 @@ def op_strlen(args: SExp): return malloc_cost(cost, args.to(size)) -def op_substr(args: SExp): +def op_substr(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: arg_count = args.list_len() if arg_count not in (2, 3): raise EvalError("substr takes exactly 2 or 3 arguments", args) @@ -269,7 +287,7 @@ def op_substr(args: SExp): return cost, args.to(s) -def op_concat(args: SExp): +def op_concat(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: cost = CONCAT_BASE_COST s = io.BytesIO() for arg in args.as_iter(): @@ -282,7 +300,7 @@ def op_concat(args: SExp): return malloc_cost(cost, args.to(r)) -def op_ash(args): +def op_ash(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: (i0, l0), (i1, l1) = args_as_int_list("ash", args, 2) if l1 > 4: raise EvalError("ash requires int32 args (with no leading zeros)", args.rest().first()) @@ -297,7 +315,7 @@ def op_ash(args): return malloc_cost(cost, args.to(r)) -def op_lsh(args): +def op_lsh(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: (i0, l0), (i1, l1) = args_as_int_list("lsh", args, 2) if l1 > 4: raise EvalError("lsh requires int32 args (with no leading zeros)", args.rest().first()) @@ -315,7 +333,7 @@ def op_lsh(args): return malloc_cost(cost, args.to(r)) -def binop_reduction(op_name, initial_value, args, op_f): +def binop_reduction(op_name: str, initial_value: int, args: _T_SExp, op_f: typing.Callable[[int, int], int]) -> typing.Tuple[int, _T_SExp]: total = initial_value arg_size = 0 cost = LOG_BASE_COST diff --git a/clvm/serialize.py b/clvm/serialize.py index d685794a..59b856e1 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -13,14 +13,20 @@ # 1000 0000 -> 0 bytes : nil # 0000 0000 -> 1 byte : zero (b'\x00') import io +import typing + from .CLVMObject import CLVMObject +from .SExp import SExp MAX_SINGLE_BYTE = 0x7F CONS_BOX_MARKER = 0xFF -def sexp_to_byte_iterator(sexp): +T = typing.TypeVar("T") + + +def sexp_to_byte_iterator(sexp: SExp) -> typing.Iterator[bytes]: todo_stack = [sexp] while todo_stack: sexp = todo_stack.pop() @@ -33,7 +39,7 @@ def sexp_to_byte_iterator(sexp): yield from atom_to_byte_iterator(sexp.as_atom()) -def atom_to_byte_iterator(as_atom): +def atom_to_byte_iterator(as_atom: bytes) -> typing.Iterator[bytes]: size = len(as_atom) if size == 0: yield b"\x80" @@ -68,18 +74,18 @@ def atom_to_byte_iterator(as_atom): ] ) else: - raise ValueError("sexp too long %s" % as_atom) + raise ValueError(f"sexp too long {as_atom!r}") yield size_blob yield as_atom -def sexp_to_stream(sexp, f): +def sexp_to_stream(sexp: SExp, f: typing.BinaryIO) -> None: for b in sexp_to_byte_iterator(sexp): f.write(b) -def _op_read_sexp(op_stack, val_stack, f, to_sexp): +def _op_read_sexp(op_stack, val_stack, f: typing.BinaryIO, to_sexp) -> None: blob = f.read(1) if len(blob) == 0: raise ValueError("bad encoding") @@ -92,13 +98,13 @@ def _op_read_sexp(op_stack, val_stack, f, to_sexp): val_stack.append(_atom_from_stream(f, b, to_sexp)) -def _op_cons(op_stack, val_stack, f, to_sexp): +def _op_cons(op_stack, val_stack, f: typing.BinaryIO, to_sexp) -> None: right = val_stack.pop() left = val_stack.pop() val_stack.append(to_sexp((left, right))) -def sexp_from_stream(f, to_sexp): +def sexp_from_stream(f: typing.BinaryIO, to_sexp: typing.Callable[..., T]) -> T: op_stack = [_op_read_sexp] val_stack = [] @@ -108,7 +114,7 @@ def sexp_from_stream(f, to_sexp): return to_sexp(val_stack.pop()) -def _op_consume_sexp(f): +def _op_consume_sexp(f: typing.BinaryIO) -> typing.Tuple[bytes, int]: blob = f.read(1) if len(blob) == 0: raise ValueError("bad encoding") @@ -118,7 +124,7 @@ def _op_consume_sexp(f): return (_consume_atom(f, b), 0) -def _consume_atom(f, b): +def _consume_atom(f: typing.BinaryIO, b: int) -> bytes: if b == 0x80: return bytes([b]) if b <= MAX_SINGLE_BYTE: @@ -132,10 +138,10 @@ def _consume_atom(f, b): bit_mask >>= 1 size_blob = bytes([ll]) if bit_count > 1: - ll = f.read(bit_count - 1) - if len(ll) != bit_count - 1: + llb = f.read(bit_count - 1) + if len(llb) != bit_count - 1: raise ValueError("bad encoding") - size_blob += ll + size_blob += llb size = int.from_bytes(size_blob, "big") if size >= 0x400000000: raise ValueError("blob too large") @@ -148,7 +154,7 @@ def _consume_atom(f, b): # instead of parsing the input stream, this function pulls out all the bytes # that represent on S-expression tree, and returns them. This is more efficient # than parsing and returning a python S-expression tree. -def sexp_buffer_from_stream(f): +def sexp_buffer_from_stream(f: typing.BinaryIO) -> bytes: ret = io.BytesIO() depth = 1 @@ -160,7 +166,7 @@ def sexp_buffer_from_stream(f): return ret.getvalue() -def _atom_from_stream(f, b, to_sexp): +def _atom_from_stream(f: typing.BinaryIO, b: int, to_sexp: typing.Callable[..., T]) -> T: if b == 0x80: return to_sexp(b"") if b <= MAX_SINGLE_BYTE: @@ -173,10 +179,10 @@ def _atom_from_stream(f, b, to_sexp): bit_mask >>= 1 size_blob = bytes([b]) if bit_count > 1: - b = f.read(bit_count - 1) - if len(b) != bit_count - 1: + bb = f.read(bit_count - 1) + if len(bb) != bit_count - 1: raise ValueError("bad encoding") - size_blob += b + size_blob += bb size = int.from_bytes(size_blob, "big") if size >= 0x400000000: raise ValueError("blob too large") diff --git a/mypy.ini b/mypy.ini index 39345d44..111ce702 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,4 +2,7 @@ files = clvm,tests,*.py ;ignore_missing_imports = True show_error_codes = True + +disallow_untyped_defs = True +no_implicit_optional = True warn_unused_ignores = True From ea40055b7948dea9137feb4dae9abf22dd88cfeb Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 10 Jul 2022 17:01:15 -0400 Subject: [PATCH 04/56] more (cherry picked from commit f2a0be43fc4c750843509e761e0865c606ca0334) --- clvm/CLVMObject.py | 9 ++++-- clvm/EvalError.py | 5 ++- clvm/as_python.py | 5 ++- clvm/more_ops.py | 16 ++++----- clvm/op_utils.py | 8 +++-- clvm/operators.py | 9 ++++-- clvm/run_program.py | 6 ++-- mypy.ini | 3 ++ tests/as_python_test.py | 66 +++++++++++++++++++------------------- tests/bls12_381_test.py | 2 +- tests/cmds_test.py | 15 +++++---- tests/operatordict_test.py | 2 +- tests/operators_test.py | 13 ++++---- tests/run_program_test.py | 2 +- tests/serialize_test.py | 61 ++++++++++++++++++----------------- tests/to_sexp_test.py | 64 ++++++++++++++++++++---------------- 16 files changed, 160 insertions(+), 126 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 81594abe..96e99aff 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import typing from typing_extensions import Protocol @@ -7,10 +9,10 @@ class CLVMObjectLike(Protocol): # It's not clear if it is possible to express the exclusivity without maybe # restructuring all the classes. atom: typing.Optional[bytes] - pair: typing.Optional[typing.Tuple["CLVMObjectLike", "CLVMObjectLike"]] + pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] -_T_CLVMObject = typing.TypeVar("_T_CLVMObject") +_T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject") class CLVMObject: @@ -29,7 +31,8 @@ class CLVMObject: @staticmethod def __new__( class_: typing.Type[_T_CLVMObject], - v: typing.Union["CLVMObject", typing.Tuple, bytes], + # v: typing.Union[CLVMObject, CLVMObjectLike, typing.Tuple[CLVMObject, CLVMObject], bytes], + v: typing.Union[typing.Tuple[CLVMObject, CLVMObject], bytes], ) -> _T_CLVMObject: if isinstance(v, class_): return v diff --git a/clvm/EvalError.py b/clvm/EvalError.py index f71f912a..3ee6a163 100644 --- a/clvm/EvalError.py +++ b/clvm/EvalError.py @@ -1,4 +1,7 @@ +from clvm.SExp import SExp + + class EvalError(Exception): - def __init__(self, message: str, sexp): + def __init__(self, message: str, sexp: SExp) -> None: super().__init__(message) self._sexp = sexp diff --git a/clvm/as_python.py b/clvm/as_python.py index 2fd42c64..099ba525 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -1,4 +1,7 @@ -def as_python(sexp): +from clvm.SExp import SExp + + +def as_python(sexp: SExp): def _roll(op_stack, val_stack): v1 = val_stack.pop() v2 = val_stack.pop() diff --git a/clvm/more_ops.py b/clvm/more_ops.py index 2f92b070..b3fc0ed0 100644 --- a/clvm/more_ops.py +++ b/clvm/more_ops.py @@ -345,7 +345,7 @@ def binop_reduction(op_name: str, initial_value: int, args: _T_SExp, op_f: typin return malloc_cost(cost, args.to(total)) -def op_logand(args): +def op_logand(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: def binop(a, b): a &= b return a @@ -353,7 +353,7 @@ def binop(a, b): return binop_reduction("logand", -1, args, binop) -def op_logior(args): +def op_logior(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: def binop(a, b): a |= b return a @@ -361,7 +361,7 @@ def binop(a, b): return binop_reduction("logior", 0, args, binop) -def op_logxor(args): +def op_logxor(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: def binop(a, b): a ^= b return a @@ -369,13 +369,13 @@ def binop(a, b): return binop_reduction("logxor", 0, args, binop) -def op_lognot(args): +def op_lognot(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: (i0, l0), = args_as_int_list("lognot", args, 1) cost = LOGNOT_BASE_COST + l0 * LOGNOT_COST_PER_BYTE return malloc_cost(cost, args.to(~i0)) -def op_not(args): +def op_not(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: (i0,) = args_as_bool_list("not", args, 1) if i0.as_atom() == b"": r = args.true @@ -385,7 +385,7 @@ def op_not(args): return cost, args.to(r) -def op_any(args): +def op_any(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: items = list(args_as_bools("any", args)) cost = BOOL_BASE_COST + len(items) * BOOL_COST_PER_ARG r = args.false @@ -396,7 +396,7 @@ def op_any(args): return cost, args.to(r) -def op_all(args): +def op_all(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: items = list(args_as_bools("all", args)) cost = BOOL_BASE_COST + len(items) * BOOL_COST_PER_ARG r = args.true @@ -407,7 +407,7 @@ def op_all(args): return cost, args.to(r) -def op_softfork(args: SExp): +def op_softfork(args: SExp) -> typing.Tuple[int, bool]: if args.list_len() < 1: raise EvalError("softfork takes at least 1 argument", args) a = args.first() diff --git a/clvm/op_utils.py b/clvm/op_utils.py index 9250b3ac..bc409a8e 100644 --- a/clvm/op_utils.py +++ b/clvm/op_utils.py @@ -1,4 +1,8 @@ -def operators_for_dict(keyword_to_atom, op_dict, op_name_lookup={}): +import types +from typing import Callable, Dict + + +def operators_for_dict(keyword_to_atom: Dict, op_dict: Dict[str, Callable], op_name_lookup: Dict = {}) -> Dict: d = {} for op in keyword_to_atom.keys(): op_name = "op_%s" % op_name_lookup.get(op, op) @@ -8,5 +12,5 @@ def operators_for_dict(keyword_to_atom, op_dict, op_name_lookup={}): return d -def operators_for_module(keyword_to_atom, mod, op_name_lookup={}): +def operators_for_module(keyword_to_atom: Dict, mod: types.ModuleType, op_name_lookup: Dict = {}) -> Dict: return operators_for_dict(keyword_to_atom, mod.__dict__, op_name_lookup) diff --git a/clvm/operators.py b/clvm/operators.py index 68686021..e7b5a7b6 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,4 +1,6 @@ -from typing import Tuple +from __future__ import annotations + +from typing import Tuple, Type, TypeVar from typing_extensions import Protocol @@ -170,6 +172,8 @@ class DefaultOperator(Protocol): def __call__(self, op: bytes, args: SExp) -> Tuple[int, SExp]: ... +_T_OperatorDict = TypeVar("_T_OperatorDict", bound="OperatorDict") + class OperatorDict(dict): """ This is a nice hack that adds `__call__` to a dictionary, so @@ -180,7 +184,8 @@ class OperatorDict(dict): quote_atom: int apply_atom: int - def __new__(class_, d: "OperatorDict", *args, **kwargs): + # TODO: how do you create an instance if that requires passing in an instance? + def __new__(class_: Type[_T_OperatorDict], d: "OperatorDict", *args: object, **kwargs) -> _T_OperatorDict: """ `quote_atom` and `apply_atom` must be set `unknown_op_handler` has a default implementation diff --git a/clvm/run_program.py b/clvm/run_program.py index f8f36471..a8b5fac0 100644 --- a/clvm/run_program.py +++ b/clvm/run_program.py @@ -1,4 +1,4 @@ -from typing import Any, Callable, List, Tuple +from typing import Any, Callable, List, Optional, Tuple from .CLVMObject import CLVMObject from .EvalError import EvalError @@ -22,7 +22,7 @@ OpStackType = List[OpCallable] -def to_pre_eval_op(pre_eval_f, to_sexp_f): +def to_pre_eval_op(pre_eval_f, to_sexp_f) -> Callable[[OpStackType, ValStackType], None]: def my_pre_eval_op(op_stack: OpStackType, value_stack: ValStackType) -> None: v = to_sexp_f(value_stack[-1]) context = pre_eval_f(v.first(), v.rest()) @@ -50,7 +50,7 @@ def run_program( program: CLVMObject, args: CLVMObject, operator_lookup: OperatorDict, - max_cost=None, + max_cost: Optional[int] = None, pre_eval_f=None, ) -> Tuple[int, SExp]: diff --git a/mypy.ini b/mypy.ini index 111ce702..5ecdcae0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,3 +6,6 @@ show_error_codes = True disallow_untyped_defs = True no_implicit_optional = True warn_unused_ignores = True + +[mypy-blspy.*] +ignore_missing_imports = True diff --git a/tests/as_python_test.py b/tests/as_python_test.py index 389c58f1..8fb41b4f 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -7,7 +7,7 @@ class dummy_class: - def __init__(self): + def __init__(self)-> None: self.i = 0 @@ -24,40 +24,40 @@ def gen_tree(depth: int) -> SExp: class AsPythonTest(unittest.TestCase): - def check_as_python(self, p): + def check_as_python(self, p) -> None: v = SExp.to(p) p1 = v.as_python() self.assertEqual(p, p1) - def test_null(self): + def test_null(self)-> None: self.check_as_python(b"") - def test_embedded_tuples(self): + def test_embedded_tuples(self)-> None: self.check_as_python((b"10", ((b"200", b"300"), b"400"))) - def test_single_bytes(self): + def test_single_bytes(self)-> None: for _ in range(256): self.check_as_python(bytes([_])) - def test_short_lists(self): + def test_short_lists(self) -> None: self.check_as_python(b"") for _ in range(256): for size in range(1, 5): self.check_as_python(bytes([_] * size)) - def test_int(self): + def test_int(self) -> None: v = SExp.to(42) self.assertEqual(v.atom, bytes([42])) - def test_none(self): + def test_none(self) -> None: v = SExp.to(None) self.assertEqual(v.atom, b"") - def test_empty_list(self): + def test_empty_list(self) -> None: v = SExp.to([]) self.assertEqual(v.atom, b"") - def test_list_of_one(self): + def test_list_of_one(self) -> None: v = SExp.to([1]) self.assertEqual(type(v.pair[0]), CLVMObject) self.assertEqual(type(v.pair[1]), CLVMObject) @@ -66,7 +66,7 @@ def test_list_of_one(self): self.assertEqual(v.pair[0].atom, b"\x01") self.assertEqual(v.pair[1].atom, b"") - def test_g1element(self): + def test_g1element(self) -> None: b = fh( "b3b8ac537f4fd6bde9b26221d49b54b17a506be147347dae5" "d081c0a6572b611d8484e338f3432971a9823976c6a232b" @@ -74,7 +74,7 @@ def test_g1element(self): v = SExp.to(G1Element(b)) self.assertEqual(v.atom, b) - def test_complex(self): + def test_complex(self) -> None: self.check_as_python((b"", b"foo")) self.check_as_python((b"", b"1")) self.check_as_python([b"2", (b"", b"1")]) @@ -83,7 +83,7 @@ def test_complex(self): [b"", b"1", b"2", [b"30", b"40", b"90"], b"600", (b"", b"18")] ) - def test_listp(self): + def test_listp(self) -> None: self.assertEqual(SExp.to(42).listp(), False) self.assertEqual(SExp.to(b"").listp(), False) self.assertEqual(SExp.to(b"1337").listp(), False) @@ -91,35 +91,35 @@ def test_listp(self): self.assertEqual(SExp.to((1337, 42)).listp(), True) self.assertEqual(SExp.to([1337, 42]).listp(), True) - def test_nullp(self): + def test_nullp(self) -> None: self.assertEqual(SExp.to(b"").nullp(), True) self.assertEqual(SExp.to(b"1337").nullp(), False) self.assertEqual(SExp.to((b"", b"")).nullp(), False) - def test_constants(self): + def test_constants(self) -> None: self.assertEqual(SExp.__null__.nullp(), True) self.assertEqual(SExp.null().nullp(), True) self.assertEqual(SExp.true, True) self.assertEqual(SExp.false, False) - def test_list_len(self): + def test_list_len(self) -> None: v = SExp.to(42) for i in range(100): self.assertEqual(v.list_len(), i) v = SExp.to((42, v)) self.assertEqual(v.list_len(), 100) - def test_list_len_atom(self): + def test_list_len_atom(self)-> None: v = SExp.to(42) self.assertEqual(v.list_len(), 0) - def test_as_int(self): + def test_as_int(self)-> None: self.assertEqual(SExp.to(fh("80")).as_int(), -128) self.assertEqual(SExp.to(fh("ff")).as_int(), -1) self.assertEqual(SExp.to(fh("0080")).as_int(), 128) self.assertEqual(SExp.to(fh("00ff")).as_int(), 255) - def test_cons(self): + def test_cons(self)-> None: # list self.assertEqual( SExp.to(H01).cons(SExp.to(H02).cons(SExp.null())).as_python(), @@ -128,10 +128,10 @@ def test_cons(self): # cons-box of two values self.assertEqual(SExp.to(H01).cons(SExp.to(H02).as_python()), (H01, H02)) - def test_string(self): + def test_string(self)-> None: self.assertEqual(SExp.to("foobar").as_atom(), b"foobar") - def test_deep_recursion(self): + def test_deep_recursion(self)-> None: d = b"2" for i in range(1000): d = [d] @@ -144,7 +144,7 @@ def test_deep_recursion(self): self.assertEqual(v.as_atom(), b"2") self.assertEqual(d, b"2") - def test_long_linked_list(self): + def test_long_linked_list(self)-> None: d = b"" for i in range(1000): d = (b"2", d) @@ -157,7 +157,7 @@ def test_long_linked_list(self): self.assertEqual(v.as_atom(), SExp.null()) self.assertEqual(d, b"") - def test_long_list(self): + def test_long_list(self)-> None: d = [1337] * 1000 v = SExp.to(d) for i in range(1000 - 1): @@ -166,14 +166,14 @@ def test_long_list(self): self.assertEqual(v.as_atom(), SExp.null()) - def test_invalid_type(self): + def test_invalid_type(self)-> None: with self.assertRaises(ValueError): s = SExp.to(dummy_class) # conversions are deferred, this is where it will fail: b = list(s.as_iter()) print(b) - def test_invalid_tuple(self): + def test_invalid_tuple(self)-> None: with self.assertRaises(ValueError): s = SExp.to((dummy_class, dummy_class)) # conversions are deferred, this is where it will fail: @@ -183,24 +183,24 @@ def test_invalid_tuple(self): with self.assertRaises(ValueError): s = SExp.to((dummy_class, dummy_class, dummy_class)) - def test_clvm_object_tuple(self): + def test_clvm_object_tuple(self)-> None: o1 = CLVMObject(b"foo") o2 = CLVMObject(b"bar") self.assertEqual(SExp.to((o1, o2)), (o1, o2)) - def test_first(self): + def test_first(self)-> None: val = SExp.to(1) self.assertRaises(EvalError, lambda: val.first()) val = SExp.to((42, val)) self.assertEqual(val.first(), SExp.to(42)) - def test_rest(self): + def test_rest(self)-> None: val = SExp.to(1) self.assertRaises(EvalError, lambda: val.rest()) val = SExp.to((42, val)) self.assertEqual(val.rest(), SExp.to(1)) - def test_as_iter(self): + def test_as_iter(self)-> None: val = list(SExp.to((1, (2, (3, (4, b""))))).as_iter()) self.assertEqual(val, [1, 2, 3, 4]) @@ -216,7 +216,7 @@ def test_as_iter(self): EvalError, lambda: list(SExp.to((1, (2, (3, (4, 5))))).as_iter()) ) - def test_eq(self): + def test_eq(self)-> None: val = SExp.to(1) self.assertTrue(val == 1) @@ -228,7 +228,7 @@ def test_eq(self): self.assertFalse(val == (1, 2)) self.assertFalse(val == (dummy_class, dummy_class)) - def test_eq_tree(self): + def test_eq_tree(self)-> None: val1 = gen_tree(2) val2 = gen_tree(2) val3 = gen_tree(3) @@ -238,14 +238,14 @@ def test_eq_tree(self): self.assertFalse(val1 == val3) self.assertFalse(val3 == val1) - def test_str(self): + def test_str(self)-> None: self.assertEqual(str(SExp.to(1)), "01") self.assertEqual(str(SExp.to(1337)), "820539") self.assertEqual(str(SExp.to(-1)), "81ff") self.assertEqual(str(gen_tree(1)), "ff820539820539") self.assertEqual(str(gen_tree(2)), "ffff820539820539ff820539820539") - def test_repr(self): + def test_repr(self)-> None: self.assertEqual(repr(SExp.to(1)), "SExp(01)") self.assertEqual(repr(SExp.to(1337)), "SExp(820539)") self.assertEqual(repr(SExp.to(-1)), "SExp(81ff)") diff --git a/tests/bls12_381_test.py b/tests/bls12_381_test.py index 18ba4919..11161c82 100644 --- a/tests/bls12_381_test.py +++ b/tests/bls12_381_test.py @@ -6,7 +6,7 @@ class BLS12_381_Test(unittest.TestCase): - def test_stream(self): + def test_stream(self) -> None: for _ in range(1, 64): p = PrivateKey.from_bytes(_.to_bytes(32, "big")).get_g1() blob = bytes(p) diff --git a/tests/cmds_test.py b/tests/cmds_test.py index c82c88dd..5494ddeb 100644 --- a/tests/cmds_test.py +++ b/tests/cmds_test.py @@ -4,6 +4,7 @@ import shlex import sys import unittest +from typing import Callable, Iterable, List, Optional, Tuple # If the REPAIR environment variable is set, any tests failing due to @@ -13,7 +14,7 @@ REPAIR = os.getenv("REPAIR", 0) -def get_test_cases(path): +def get_test_cases(path: str) -> List[Tuple[str, List[str], str, List[str], str]]: PREFIX = os.path.dirname(__file__) TESTS_PATH = os.path.join(PREFIX, path) paths = [] @@ -44,7 +45,7 @@ def get_test_cases(path): class TestCmds(unittest.TestCase): - def invoke_tool(self, cmd_line): + def invoke_tool(self, cmd_line: str) -> Tuple[Optional[int], str, str]: # capture io stdout_buffer = io.StringIO() @@ -57,7 +58,7 @@ def invoke_tool(self, cmd_line): sys.stderr = stderr_buffer args = shlex.split(cmd_line) - v = pkg_resources.load_entry_point("clvm_tools", "console_scripts", args[0])( + v: Optional[int] = pkg_resources.load_entry_point("clvm_tools", "console_scripts", args[0])( args ) @@ -67,8 +68,8 @@ def invoke_tool(self, cmd_line): return v, stdout_buffer.getvalue(), stderr_buffer.getvalue() -def make_f(cmd_lines, expected_output, comments, path): - def f(self): +def make_f(cmd_lines: List[str], expected_output: object, comments: Iterable[str], path: str) -> Callable[[TestCmds], None]: + def f(self: TestCmds) -> None: cmd = "".join(cmd_lines) for c in cmd.split(";"): r, actual_output, actual_stderr = self.invoke_tool(c) @@ -92,7 +93,7 @@ def f(self): return f -def inject(*paths): +def inject(*paths: str) -> None: for path in paths: for idx, (name, i, o, comments, path) in enumerate(get_test_cases(path)): name_of_f = "test_%s" % name @@ -104,7 +105,7 @@ def inject(*paths): inject("unknown-op") -def main(): +def main() -> None: unittest.main() diff --git a/tests/operatordict_test.py b/tests/operatordict_test.py index 897f6ffe..830dba5a 100644 --- a/tests/operatordict_test.py +++ b/tests/operatordict_test.py @@ -4,7 +4,7 @@ class OperatorDictTest(unittest.TestCase): - def test_operatordict_constructor(self): + def test_operatordict_constructor(self) -> None: """Constructing should fail if quote or apply are not specified, either by object property or by keyword argument. Note that they cannot be specified in the operator dictionary itself. diff --git a/tests/operators_test.py b/tests/operators_test.py index 9c84d719..573e6a31 100644 --- a/tests/operators_test.py +++ b/tests/operators_test.py @@ -1,4 +1,5 @@ import unittest +from typing import Tuple from clvm.operators import (OPERATOR_LOOKUP, KEYWORD_TO_ATOM, default_unknown_op, OperatorDict) from clvm.EvalError import EvalError @@ -8,16 +9,16 @@ class OperatorsTest(unittest.TestCase): - def setUp(self): + def setUp(self)-> None: self.handler_called = False - def unknown_handler(self, name, args): + def unknown_handler(self, name: bytes, args: SExp) -> Tuple[int, SExp]: self.handler_called = True self.assertEqual(name, b'\xff\xff1337') self.assertEqual(args, SExp.to(1337)) return 42, SExp.to(b'foobar') - def test_unknown_op(self): + def test_unknown_op(self)-> None: self.assertRaises(EvalError, lambda: OPERATOR_LOOKUP(b'\xff\xff1337', SExp.to(1337))) od = OperatorDict(OPERATOR_LOOKUP, unknown_op_handler=lambda name, args: self.unknown_handler(name, args)) cost, ret = od(b'\xff\xff1337', SExp.to(1337)) @@ -25,11 +26,11 @@ def test_unknown_op(self): self.assertEqual(cost, 42) self.assertEqual(ret, SExp.to(b'foobar')) - def test_plus(self): + def test_plus(self)-> None: print(OPERATOR_LOOKUP) self.assertEqual(OPERATOR_LOOKUP(KEYWORD_TO_ATOM['+'], SExp.to([3, 4, 5]))[1], SExp.to(12)) - def test_unknown_op_reserved(self): + def test_unknown_op_reserved(self)-> None: # any op that starts with ffff is reserved, and results in a hard # failure @@ -51,7 +52,7 @@ def test_unknown_op_reserved(self): # the cost is 0xffff00 = 16776960 self.assertEqual(default_unknown_op(b"\x00\xff\xff\x00\x00", SExp.null()), (16776961, SExp.null())) - def test_unknown_ops_last_bits(self): + def test_unknown_ops_last_bits(self)-> None: # The last byte is ignored for no-op unknown ops for suffix in [b"\x3f", b"\x0f", b"\x00", b"\x2c"]: diff --git a/tests/run_program_test.py b/tests/run_program_test.py index d64462ca..78ffb998 100644 --- a/tests/run_program_test.py +++ b/tests/run_program_test.py @@ -5,7 +5,7 @@ class BitTest(unittest.TestCase): - def test_msb_mask(self): + def test_msb_mask(self) -> None: self.assertEqual(msb_mask(0x0), 0x0) self.assertEqual(msb_mask(0x01), 0x01) self.assertEqual(msb_mask(0x02), 0x02) diff --git a/tests/serialize_test.py b/tests/serialize_test.py index 786f1c95..d2130a23 100644 --- a/tests/serialize_test.py +++ b/tests/serialize_test.py @@ -1,5 +1,7 @@ import io +import math import unittest +from typing import Optional from clvm import to_sexp_f from clvm.serialize import (sexp_from_stream, sexp_buffer_from_stream, atom_to_byte_iterator) @@ -8,32 +10,29 @@ TEXT = b"the quick brown fox jumps over the lazy dogs" -class InfiniteStream(io.TextIOBase): - def __init__(self, b): - self.buf = b +class InfiniteStream(io.BytesIO): + def read(self, n: Optional[int] = -1) -> bytes: + result = super().read(n) - def read(self, n): - ret = b'' - while n > 0 and len(self.buf) > 0: - ret += self.buf[0:1] - self.buf = self.buf[1:] - n -= 1 - ret += b' ' * n - return ret + if n is not None and n > 0: + fill_needed = n - len(result) + result += b' ' * fill_needed + + return result class LargeAtom: - def __len__(self): + def __len__(self) -> int: return 0x400000001 class SerializeTest(unittest.TestCase): - def check_serde(self, s): + def check_serde(self, s) -> None: v = to_sexp_f(s) b = v.as_bin() v1 = sexp_from_stream(io.BytesIO(b), to_sexp_f) if v != v1: - print("%s: %d %s %s" % (v, len(b), b, v1)) + print("%s: %d %r %s" % (v, len(b), b, v1)) breakpoint() b = v.as_bin() v1 = sexp_from_stream(io.BytesIO(b), to_sexp_f) @@ -44,44 +43,46 @@ def check_serde(self, s): buf = sexp_buffer_from_stream(io.BytesIO(b)) self.assertEqual(buf, b) - def test_zero(self): + def test_zero(self) -> None: v = to_sexp_f(b"\x00") self.assertEqual(v.as_bin(), b"\x00") - def test_empty(self): + def test_empty(self) -> None: v = to_sexp_f(b"") self.assertEqual(v.as_bin(), b"\x80") - def test_empty_string(self): + def test_empty_string(self) -> None: self.check_serde(b"") - def test_single_bytes(self): + def test_single_bytes(self) -> None: for _ in range(256): self.check_serde(bytes([_])) - def test_short_lists(self): + def test_short_lists(self) -> None: self.check_serde([]) for _ in range(0, 2048, 8): for size in range(1, 5): self.check_serde([_] * size) - def test_cons_box(self): + def test_cons_box(self) -> None: self.check_serde((None, None)) self.check_serde((None, [1, 2, 30, 40, 600, (None, 18)])) self.check_serde((100, (TEXT, (30, (50, (90, (TEXT, TEXT + TEXT))))))) - def test_long_blobs(self): + def test_long_blobs(self) -> None: text = TEXT * 300 for _, t in enumerate(text): t1 = text[:_] self.check_serde(t1) - def test_blob_limit(self): + def test_blob_limit(self) -> None: with self.assertRaises(ValueError): - for b in atom_to_byte_iterator(LargeAtom()): - print('%02x' % b) + # Specifically substituting another type that is sufficiently similar to + # the expected bytes for this test. + for _ in atom_to_byte_iterator(LargeAtom()): # type: ignore[arg-type] + pass - def test_very_long_blobs(self): + def test_very_long_blobs(self) -> None: for size in [0x40, 0x2000, 0x100000, 0x8000000]: count = size // len(TEXT) text = TEXT * count @@ -91,7 +92,7 @@ def test_very_long_blobs(self): assert len(text) > size self.check_serde(text) - def test_very_deep_tree(self): + def test_very_deep_tree(self) -> None: blob = b"a" for depth in [10, 100, 1000, 10000, 100000]: s = to_sexp_f(blob) @@ -99,7 +100,7 @@ def test_very_deep_tree(self): s = to_sexp_f((s, blob)) self.check_serde(s) - def test_deserialize_empty(self): + def test_deserialize_empty(self) -> None: bytes_in = b'' with self.assertRaises(ValueError): sexp_from_stream(io.BytesIO(bytes_in), to_sexp_f) @@ -107,7 +108,7 @@ def test_deserialize_empty(self): with self.assertRaises(ValueError): sexp_buffer_from_stream(io.BytesIO(bytes_in)) - def test_deserialize_truncated_size(self): + def test_deserialize_truncated_size(self) -> None: # fe means the total number of bytes in the length-prefix is 7 # one for each bit set. 5 bytes is too few bytes_in = b'\xfe ' @@ -117,7 +118,7 @@ def test_deserialize_truncated_size(self): with self.assertRaises(ValueError): sexp_buffer_from_stream(io.BytesIO(bytes_in)) - def test_deserialize_truncated_blob(self): + def test_deserialize_truncated_blob(self) -> None: # this is a complete length prefix. The blob is supposed to be 63 bytes # the blob itself is truncated though, it's less than 63 bytes bytes_in = b'\xbf ' @@ -128,7 +129,7 @@ def test_deserialize_truncated_blob(self): with self.assertRaises(ValueError): sexp_buffer_from_stream(io.BytesIO(bytes_in)) - def test_deserialize_large_blob(self): + def test_deserialize_large_blob(self) -> None: # this length prefix is 7 bytes long, the last 6 bytes specifies the # length of the blob, which is 0xffffffffffff, or (2^48 - 1) # we don't support blobs this large, and we should fail immediately when diff --git a/tests/to_sexp_test.py b/tests/to_sexp_test.py index 36a73e49..26209567 100644 --- a/tests/to_sexp_test.py +++ b/tests/to_sexp_test.py @@ -1,11 +1,12 @@ import unittest +from dataclasses import dataclass from typing import Optional, Tuple, Any from clvm.SExp import SExp, looks_like_clvm_object, convert_atom_to_bytes from clvm.CLVMObject import CLVMObject -def validate_sexp(sexp): +def validate_sexp(sexp: SExp) -> None: validate_stack = [sexp] while validate_stack: v = validate_stack.pop() @@ -50,22 +51,38 @@ def print_tree(tree: SExp) -> str: return ret +@dataclass(frozen=True) +class PairAndAtom: + pair: None = None + atom: None = None + + +@dataclass(frozen=True) +class Pair: + pair: None = None + + +@dataclass(frozen=True) +class Atom: + atom: None = None + + class ToSExpTest(unittest.TestCase): - def test_cast_1(self): + def test_cast_1(self) -> None: # this was a problem in `clvm_tools` and is included # to prevent regressions sexp = SExp.to(b"foo") t1 = sexp.to([1, sexp]) validate_sexp(t1) - def test_wrap_sexp(self): + def test_wrap_sexp(self) -> None: # it's a bit of a layer violation that CLVMObject unwraps SExp, but we # rely on that in a fair number of places for now. We should probably # work towards phasing that out o = CLVMObject(SExp.to(1)) assert o.atom == bytes([1]) - def test_arbitrary_underlying_tree(self): + def test_arbitrary_underlying_tree(self) -> None: # SExp provides a view on top of a tree of arbitrary types, as long as # those types implement the CLVMObject protocol. This is an example of @@ -75,7 +92,7 @@ class GeneratedTree: depth: int = 4 val: int = 0 - def __init__(self, depth, val): + def __init__(self, depth: int, val: int) -> None: assert depth >= 0 self.depth = depth self.val = val @@ -103,52 +120,45 @@ def pair(self) -> Optional[Tuple[Any, Any]]: tree = SExp.to(GeneratedTree(3, 10)) assert print_leaves(tree) == "10 11 12 13 14 15 16 17 " - def test_looks_like_clvm_object(self): + def test_looks_like_clvm_object(self) -> None: # this function can't look at the values, that would cause a cascade of # eager evaluation/conversion - class dummy: - pass - - obj = dummy() - obj.pair = None - obj.atom = None - print(dir(obj)) - assert looks_like_clvm_object(obj) + pair_and_atom = PairAndAtom() + print(dir(pair_and_atom)) + assert looks_like_clvm_object(pair_and_atom) - obj = dummy() - obj.pair = None - assert not looks_like_clvm_object(obj) + pair = Pair() + assert not looks_like_clvm_object(pair) - obj = dummy() - obj.atom = None - assert not looks_like_clvm_object(obj) + atom = Atom() + assert not looks_like_clvm_object(atom) - def test_list_conversions(self): + def test_list_conversions(self) -> None: a = SExp.to([1, 2, 3]) assert print_tree(a) == "(1 (2 (3 () )))" - def test_string_conversions(self): + def test_string_conversions(self) -> None: a = SExp.to("foobar") assert a.as_atom() == "foobar".encode() - def test_int_conversions(self): + def test_int_conversions(self) -> None: a = SExp.to(1337) assert a.as_atom() == bytes([0x5, 0x39]) - def test_none_conversions(self): + def test_none_conversions(self) -> None: a = SExp.to(None) assert a.as_atom() == b"" - def test_empty_list_conversions(self): + def test_empty_list_conversions(self) -> None: a = SExp.to([]) assert a.as_atom() == b"" - def test_eager_conversion(self): + def test_eager_conversion(self) -> None: with self.assertRaises(ValueError): SExp.to(("foobar", (1, {}))) - def test_convert_atom(self): + def test_convert_atom(self) -> None: assert convert_atom_to_bytes(0x133742) == bytes([0x13, 0x37, 0x42]) assert convert_atom_to_bytes(0x833742) == bytes([0x00, 0x83, 0x37, 0x42]) assert convert_atom_to_bytes(0) == b"" From bf598ff3f59cf373d8e98f5742e6dbce6481bf62 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 17 Aug 2022 21:31:37 -0400 Subject: [PATCH 05/56] fixup --- clvm/SExp.py | 3 +++ clvm/more_ops.py | 9 +++++++-- clvm/operators.py | 1 + tests/as_python_test.py | 42 ++++++++++++++++++++--------------------- tests/cmds_test.py | 7 ++++++- tests/operators_test.py | 10 +++++----- tests/serialize_test.py | 1 - 7 files changed, 43 insertions(+), 30 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index f5463334..5a777b11 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -118,6 +118,9 @@ def to_sexp_type( return stack[0] +_T_SExp = typing.TypeVar("_T_SExp") + + # TODO: Maybe there is some way to track atom vs. pair SExps to help hinting out a bit class SExp: """ diff --git a/clvm/more_ops.py b/clvm/more_ops.py index b3fc0ed0..b045388b 100644 --- a/clvm/more_ops.py +++ b/clvm/more_ops.py @@ -220,7 +220,7 @@ def op_gr_bytes(args: SExp) -> typing.Tuple[int, SExp]: b1 = a1.as_atom() cost = GRS_BASE_COST if b0 is None or b1 is None: - raise TypeError(f"Internal error, both operands must not be None") + raise TypeError("Internal error, both operands must not be None") cost += (len(b0) + len(b1)) * GRS_COST_PER_BYTE return cost, args.true if b0 > b1 else args.false @@ -333,7 +333,12 @@ def op_lsh(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: return malloc_cost(cost, args.to(r)) -def binop_reduction(op_name: str, initial_value: int, args: _T_SExp, op_f: typing.Callable[[int, int], int]) -> typing.Tuple[int, _T_SExp]: +def binop_reduction( + op_name: str, + initial_value: int, + args: _T_SExp, + op_f: typing.Callable[[int, int], int], +) -> typing.Tuple[int, _T_SExp]: total = initial_value arg_size = 0 cost = LOG_BASE_COST diff --git a/clvm/operators.py b/clvm/operators.py index e7b5a7b6..2f949a4d 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -174,6 +174,7 @@ def __call__(self, op: bytes, args: SExp) -> Tuple[int, SExp]: ... _T_OperatorDict = TypeVar("_T_OperatorDict", bound="OperatorDict") + class OperatorDict(dict): """ This is a nice hack that adds `__call__` to a dictionary, so diff --git a/tests/as_python_test.py b/tests/as_python_test.py index 8fb41b4f..edbc5155 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -7,7 +7,7 @@ class dummy_class: - def __init__(self)-> None: + def __init__(self) -> None: self.i = 0 @@ -29,13 +29,13 @@ def check_as_python(self, p) -> None: p1 = v.as_python() self.assertEqual(p, p1) - def test_null(self)-> None: + def test_null(self) -> None: self.check_as_python(b"") - def test_embedded_tuples(self)-> None: + def test_embedded_tuples(self) -> None: self.check_as_python((b"10", ((b"200", b"300"), b"400"))) - def test_single_bytes(self)-> None: + def test_single_bytes(self) -> None: for _ in range(256): self.check_as_python(bytes([_])) @@ -109,17 +109,17 @@ def test_list_len(self) -> None: v = SExp.to((42, v)) self.assertEqual(v.list_len(), 100) - def test_list_len_atom(self)-> None: + def test_list_len_atom(self) -> None: v = SExp.to(42) self.assertEqual(v.list_len(), 0) - def test_as_int(self)-> None: + def test_as_int(self) -> None: self.assertEqual(SExp.to(fh("80")).as_int(), -128) self.assertEqual(SExp.to(fh("ff")).as_int(), -1) self.assertEqual(SExp.to(fh("0080")).as_int(), 128) self.assertEqual(SExp.to(fh("00ff")).as_int(), 255) - def test_cons(self)-> None: + def test_cons(self) -> None: # list self.assertEqual( SExp.to(H01).cons(SExp.to(H02).cons(SExp.null())).as_python(), @@ -128,10 +128,10 @@ def test_cons(self)-> None: # cons-box of two values self.assertEqual(SExp.to(H01).cons(SExp.to(H02).as_python()), (H01, H02)) - def test_string(self)-> None: + def test_string(self) -> None: self.assertEqual(SExp.to("foobar").as_atom(), b"foobar") - def test_deep_recursion(self)-> None: + def test_deep_recursion(self) -> None: d = b"2" for i in range(1000): d = [d] @@ -144,7 +144,7 @@ def test_deep_recursion(self)-> None: self.assertEqual(v.as_atom(), b"2") self.assertEqual(d, b"2") - def test_long_linked_list(self)-> None: + def test_long_linked_list(self) -> None: d = b"" for i in range(1000): d = (b"2", d) @@ -157,7 +157,7 @@ def test_long_linked_list(self)-> None: self.assertEqual(v.as_atom(), SExp.null()) self.assertEqual(d, b"") - def test_long_list(self)-> None: + def test_long_list(self) -> None: d = [1337] * 1000 v = SExp.to(d) for i in range(1000 - 1): @@ -166,14 +166,14 @@ def test_long_list(self)-> None: self.assertEqual(v.as_atom(), SExp.null()) - def test_invalid_type(self)-> None: + def test_invalid_type(self) -> None: with self.assertRaises(ValueError): s = SExp.to(dummy_class) # conversions are deferred, this is where it will fail: b = list(s.as_iter()) print(b) - def test_invalid_tuple(self)-> None: + def test_invalid_tuple(self) -> None: with self.assertRaises(ValueError): s = SExp.to((dummy_class, dummy_class)) # conversions are deferred, this is where it will fail: @@ -183,24 +183,24 @@ def test_invalid_tuple(self)-> None: with self.assertRaises(ValueError): s = SExp.to((dummy_class, dummy_class, dummy_class)) - def test_clvm_object_tuple(self)-> None: + def test_clvm_object_tuple(self) -> None: o1 = CLVMObject(b"foo") o2 = CLVMObject(b"bar") self.assertEqual(SExp.to((o1, o2)), (o1, o2)) - def test_first(self)-> None: + def test_first(self) -> None: val = SExp.to(1) self.assertRaises(EvalError, lambda: val.first()) val = SExp.to((42, val)) self.assertEqual(val.first(), SExp.to(42)) - def test_rest(self)-> None: + def test_rest(self) -> None: val = SExp.to(1) self.assertRaises(EvalError, lambda: val.rest()) val = SExp.to((42, val)) self.assertEqual(val.rest(), SExp.to(1)) - def test_as_iter(self)-> None: + def test_as_iter(self) -> None: val = list(SExp.to((1, (2, (3, (4, b""))))).as_iter()) self.assertEqual(val, [1, 2, 3, 4]) @@ -216,7 +216,7 @@ def test_as_iter(self)-> None: EvalError, lambda: list(SExp.to((1, (2, (3, (4, 5))))).as_iter()) ) - def test_eq(self)-> None: + def test_eq(self) -> None: val = SExp.to(1) self.assertTrue(val == 1) @@ -228,7 +228,7 @@ def test_eq(self)-> None: self.assertFalse(val == (1, 2)) self.assertFalse(val == (dummy_class, dummy_class)) - def test_eq_tree(self)-> None: + def test_eq_tree(self) -> None: val1 = gen_tree(2) val2 = gen_tree(2) val3 = gen_tree(3) @@ -238,14 +238,14 @@ def test_eq_tree(self)-> None: self.assertFalse(val1 == val3) self.assertFalse(val3 == val1) - def test_str(self)-> None: + def test_str(self) -> None: self.assertEqual(str(SExp.to(1)), "01") self.assertEqual(str(SExp.to(1337)), "820539") self.assertEqual(str(SExp.to(-1)), "81ff") self.assertEqual(str(gen_tree(1)), "ff820539820539") self.assertEqual(str(gen_tree(2)), "ffff820539820539ff820539820539") - def test_repr(self)-> None: + def test_repr(self) -> None: self.assertEqual(repr(SExp.to(1)), "SExp(01)") self.assertEqual(repr(SExp.to(1337)), "SExp(820539)") self.assertEqual(repr(SExp.to(-1)), "SExp(81ff)") diff --git a/tests/cmds_test.py b/tests/cmds_test.py index 5494ddeb..4073e635 100644 --- a/tests/cmds_test.py +++ b/tests/cmds_test.py @@ -68,7 +68,12 @@ def invoke_tool(self, cmd_line: str) -> Tuple[Optional[int], str, str]: return v, stdout_buffer.getvalue(), stderr_buffer.getvalue() -def make_f(cmd_lines: List[str], expected_output: object, comments: Iterable[str], path: str) -> Callable[[TestCmds], None]: +def make_f( + cmd_lines: List[str], + expected_output: object, + comments: Iterable[str], + path: str, +) -> Callable[[TestCmds], None]: def f(self: TestCmds) -> None: cmd = "".join(cmd_lines) for c in cmd.split(";"): diff --git a/tests/operators_test.py b/tests/operators_test.py index 573e6a31..443a42da 100644 --- a/tests/operators_test.py +++ b/tests/operators_test.py @@ -9,7 +9,7 @@ class OperatorsTest(unittest.TestCase): - def setUp(self)-> None: + def setUp(self) -> None: self.handler_called = False def unknown_handler(self, name: bytes, args: SExp) -> Tuple[int, SExp]: @@ -18,7 +18,7 @@ def unknown_handler(self, name: bytes, args: SExp) -> Tuple[int, SExp]: self.assertEqual(args, SExp.to(1337)) return 42, SExp.to(b'foobar') - def test_unknown_op(self)-> None: + def test_unknown_op(self) -> None: self.assertRaises(EvalError, lambda: OPERATOR_LOOKUP(b'\xff\xff1337', SExp.to(1337))) od = OperatorDict(OPERATOR_LOOKUP, unknown_op_handler=lambda name, args: self.unknown_handler(name, args)) cost, ret = od(b'\xff\xff1337', SExp.to(1337)) @@ -26,11 +26,11 @@ def test_unknown_op(self)-> None: self.assertEqual(cost, 42) self.assertEqual(ret, SExp.to(b'foobar')) - def test_plus(self)-> None: + def test_plus(self) -> None: print(OPERATOR_LOOKUP) self.assertEqual(OPERATOR_LOOKUP(KEYWORD_TO_ATOM['+'], SExp.to([3, 4, 5]))[1], SExp.to(12)) - def test_unknown_op_reserved(self)-> None: + def test_unknown_op_reserved(self) -> None: # any op that starts with ffff is reserved, and results in a hard # failure @@ -52,7 +52,7 @@ def test_unknown_op_reserved(self)-> None: # the cost is 0xffff00 = 16776960 self.assertEqual(default_unknown_op(b"\x00\xff\xff\x00\x00", SExp.null()), (16776961, SExp.null())) - def test_unknown_ops_last_bits(self)-> None: + def test_unknown_ops_last_bits(self) -> None: # The last byte is ignored for no-op unknown ops for suffix in [b"\x3f", b"\x0f", b"\x00", b"\x2c"]: diff --git a/tests/serialize_test.py b/tests/serialize_test.py index d2130a23..6753b79b 100644 --- a/tests/serialize_test.py +++ b/tests/serialize_test.py @@ -1,5 +1,4 @@ import io -import math import unittest from typing import Optional From 2293297d326e8580c91cf14099b05384956a1baa Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 17 Aug 2022 21:52:16 -0400 Subject: [PATCH 06/56] little --- clvm/CLVMObject.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 96e99aff..3b23551f 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -31,6 +31,7 @@ class CLVMObject: @staticmethod def __new__( class_: typing.Type[_T_CLVMObject], + # TODO: which? review? # v: typing.Union[CLVMObject, CLVMObjectLike, typing.Tuple[CLVMObject, CLVMObject], bytes], v: typing.Union[typing.Tuple[CLVMObject, CLVMObject], bytes], ) -> _T_CLVMObject: @@ -39,10 +40,7 @@ def __new__( self = super(CLVMObject, class_).__new__(class_) if isinstance(v, tuple): if len(v) != 2: - raise ValueError( - "tuples must be of size 2, cannot create CLVMObject from: %s" - % str(v) - ) + raise ValueError("tuples must be of size 2, cannot create CLVMObject from: %s" % str(v)) self.pair = v self.atom = None else: From 3a5e0b02cdefd8fdd9c5593ac5963a2b0636dc9e Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Nov 2023 09:43:50 -0500 Subject: [PATCH 07/56] catch up on test hints --- clvm/SExp.py | 2 +- clvm/operators.py | 6 ++--- tests/as_python_test.py | 50 ++++++++++++++++++++++++++++------------- tests/serialize_test.py | 3 ++- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index 8346edda..b2634514 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -22,7 +22,7 @@ int, None, list, - typing.Tuple["CastableType", "CastableType"], + typing.Tuple["CastableType", ...], ] diff --git a/clvm/operators.py b/clvm/operators.py index 2f949a4d..d20de8fb 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Tuple, Type, TypeVar +from typing import Dict, Tuple, Type, TypeVar from typing_extensions import Protocol @@ -186,14 +186,14 @@ class OperatorDict(dict): apply_atom: int # TODO: how do you create an instance if that requires passing in an instance? - def __new__(class_: Type[_T_OperatorDict], d: "OperatorDict", *args: object, **kwargs) -> _T_OperatorDict: + def __new__(cls: Type[_T_OperatorDict], d: Dict, *args: object, **kwargs) -> _T_OperatorDict: """ `quote_atom` and `apply_atom` must be set `unknown_op_handler` has a default implementation We do not check if quote and apply are distinct We do not check if the opcode values for quote and apply exist in the passed-in dict """ - self = super(OperatorDict, class_).__new__(class_, d) + self = super().__new__(cls, d) self.quote_atom = kwargs["quote"] if "quote" in kwargs else d.quote_atom self.apply_atom = kwargs["apply"] if "apply" in kwargs else d.apply_atom if "unknown_op_handler" in kwargs: diff --git a/tests/as_python_test.py b/tests/as_python_test.py index edbc5155..06f4fff5 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -1,7 +1,10 @@ import unittest +from typing import List, Tuple, Union + from clvm import SExp from clvm.CLVMObject import CLVMObject +from clvm.SExp import CastableType from blspy import G1Element from clvm.EvalError import EvalError @@ -22,9 +25,12 @@ def gen_tree(depth: int) -> SExp: H01 = fh("01") H02 = fh("02") +NestedListOfBytes = Union[bytes, List["NestedListOfBytes"]] +NestedTupleOfBytes = Union[bytes, Tuple["NestedTupleOfBytes", ...]] + class AsPythonTest(unittest.TestCase): - def check_as_python(self, p) -> None: + def check_as_python(self, p: CastableType) -> None: v = SExp.to(p) p1 = v.as_python() self.assertEqual(p, p1) @@ -59,10 +65,13 @@ def test_empty_list(self) -> None: def test_list_of_one(self) -> None: v = SExp.to([1]) + assert v.pair is not None self.assertEqual(type(v.pair[0]), CLVMObject) self.assertEqual(type(v.pair[1]), CLVMObject) - self.assertEqual(type(v.as_pair()[0]), SExp) - self.assertEqual(type(v.as_pair()[1]), SExp) + from_as_pair = v.as_pair() + assert from_as_pair is not None + self.assertEqual(type(from_as_pair[0]), SExp) + self.assertEqual(type(from_as_pair[1]), SExp) self.assertEqual(v.pair[0].atom, b"\x01") self.assertEqual(v.pair[1].atom, b"") @@ -132,26 +141,35 @@ def test_string(self) -> None: self.assertEqual(SExp.to("foobar").as_atom(), b"foobar") def test_deep_recursion(self) -> None: - d = b"2" + d: NestedListOfBytes = b"2" for i in range(1000): d = [d] v = SExp.to(d) for i in range(1000): - self.assertEqual(v.as_pair()[1].as_atom(), SExp.null()) - v = v.as_pair()[0] - d = d[0] + from_as_pair = v.as_pair() + assert from_as_pair is not None + self.assertEqual(from_as_pair[1].as_atom(), SExp.null()) + v = from_as_pair[0] + element = d[0] + assert isinstance(element, list) + d = element self.assertEqual(v.as_atom(), b"2") self.assertEqual(d, b"2") def test_long_linked_list(self) -> None: - d = b"" + d: NestedTupleOfBytes = b"" for i in range(1000): d = (b"2", d) v = SExp.to(d) for i in range(1000): - self.assertEqual(v.as_pair()[0].as_atom(), d[0]) - v = v.as_pair()[1] + from_as_pair = v.as_pair() + assert from_as_pair is not None + self.assertEqual(from_as_pair[0].as_atom(), d[0]) + from_as_pair = v.as_pair() + assert from_as_pair is not None + v = from_as_pair[1] + assert isinstance(d, tuple) d = d[1] self.assertEqual(v.as_atom(), SExp.null()) @@ -161,27 +179,29 @@ def test_long_list(self) -> None: d = [1337] * 1000 v = SExp.to(d) for i in range(1000 - 1): - self.assertEqual(v.as_pair()[0].as_int(), d[i]) - v = v.as_pair()[1] + from_as_pair = v.as_pair() + assert from_as_pair is not None + self.assertEqual(from_as_pair[0].as_int(), d[i]) + v = from_as_pair[1] self.assertEqual(v.as_atom(), SExp.null()) def test_invalid_type(self) -> None: + s = SExp.to(dummy_class) # type: ignore[arg-type] with self.assertRaises(ValueError): - s = SExp.to(dummy_class) # conversions are deferred, this is where it will fail: b = list(s.as_iter()) print(b) def test_invalid_tuple(self) -> None: + s = SExp.to((dummy_class, dummy_class)) # type: ignore[arg-type] with self.assertRaises(ValueError): - s = SExp.to((dummy_class, dummy_class)) # conversions are deferred, this is where it will fail: b = list(s.as_iter()) print(b) with self.assertRaises(ValueError): - s = SExp.to((dummy_class, dummy_class, dummy_class)) + s = SExp.to((dummy_class, dummy_class, dummy_class)) # type: ignore[arg-type] def test_clvm_object_tuple(self) -> None: o1 = CLVMObject(b"foo") diff --git a/tests/serialize_test.py b/tests/serialize_test.py index 6753b79b..250b202b 100644 --- a/tests/serialize_test.py +++ b/tests/serialize_test.py @@ -3,6 +3,7 @@ from typing import Optional from clvm import to_sexp_f +from clvm.SExp import CastableType from clvm.serialize import (sexp_from_stream, sexp_buffer_from_stream, atom_to_byte_iterator) @@ -26,7 +27,7 @@ def __len__(self) -> int: class SerializeTest(unittest.TestCase): - def check_serde(self, s) -> None: + def check_serde(self, s: CastableType) -> None: v = to_sexp_f(s) b = v.as_bin() v1 = sexp_from_stream(io.BytesIO(b), to_sexp_f) From 0f3f7a7f1377ec8866731ccecbb9b2453f44de9b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Nov 2023 10:29:29 -0500 Subject: [PATCH 08/56] more --- clvm/CLVMObject.py | 3 ++- clvm/EvalError.py | 7 ++++++- clvm/SExp.py | 6 +++--- clvm/as_python.py | 7 ++++++- clvm/core_ops.py | 21 ++++++++++++++------- clvm/operators.py | 8 ++++---- clvm/serialize.py | 7 ++++++- tests/operatordict_test.py | 7 +++++-- tests/to_sexp_test.py | 34 +++++++++++++++++++++++++--------- 9 files changed, 71 insertions(+), 29 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 3b23551f..d4dfcc04 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -8,6 +8,7 @@ class CLVMObjectLike(Protocol): # It's not clear if it is possible to express the exclusivity without maybe # restructuring all the classes. + # TODO: can we make these read only? atom: typing.Optional[bytes] pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] @@ -33,7 +34,7 @@ def __new__( class_: typing.Type[_T_CLVMObject], # TODO: which? review? # v: typing.Union[CLVMObject, CLVMObjectLike, typing.Tuple[CLVMObject, CLVMObject], bytes], - v: typing.Union[typing.Tuple[CLVMObject, CLVMObject], bytes], + v: typing.Union[CLVMObjectLike, bytes], ) -> _T_CLVMObject: if isinstance(v, class_): return v diff --git a/clvm/EvalError.py b/clvm/EvalError.py index 3ee6a163..e8e31482 100644 --- a/clvm/EvalError.py +++ b/clvm/EvalError.py @@ -1,4 +1,9 @@ -from clvm.SExp import SExp +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from clvm.SExp import SExp class EvalError(Exception): diff --git a/clvm/SExp.py b/clvm/SExp.py index b2634514..7875b46e 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -16,7 +16,7 @@ CastableType = typing.Union[ "SExp", - "CLVMObject", + CLVMObjectLike, bytes, str, int, @@ -36,7 +36,7 @@ def looks_like_clvm_object(o: typing.Any) -> bool: # this function recognizes some common types and turns them into plain bytes, def convert_atom_to_bytes( - v: typing.Union[bytes, str, int, None, list], + v: typing.Union[bytes, str, int, None, typing.List, typing.SupportsBytes], ) -> bytes: if isinstance(v, bytes): @@ -116,7 +116,7 @@ def to_sexp_type( return stack[0] -_T_SExp = typing.TypeVar("_T_SExp") +_T_SExp = typing.TypeVar("_T_SExp", bound="SExp") # TODO: Maybe there is some way to track atom vs. pair SExps to help hinting out a bit diff --git a/clvm/as_python.py b/clvm/as_python.py index 099ba525..9370d21d 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -1,4 +1,9 @@ -from clvm.SExp import SExp +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from clvm.SExp import SExp def as_python(sexp: SExp): diff --git a/clvm/core_ops.py b/clvm/core_ops.py index 850e5ce7..3a114125 100644 --- a/clvm/core_ops.py +++ b/clvm/core_ops.py @@ -1,3 +1,5 @@ +from typing import Tuple, TypeVar + from .EvalError import EvalError from .SExp import SExp @@ -12,7 +14,12 @@ ) -def op_if(args: SExp): +_T_SExp = TypeVar("_T_SExp", bound=SExp) + +# TODO: should all the int return types be more specific integer types? + + +def op_if(args: _T_SExp) -> Tuple[int, _T_SExp]: if args.list_len() != 3: raise EvalError("i takes exactly 3 arguments", args) r = args.rest() @@ -21,38 +28,38 @@ def op_if(args: SExp): return IF_COST, r.first() -def op_cons(args: SExp): +def op_cons(args: _T_SExp) -> Tuple[int, _T_SExp]: if args.list_len() != 2: raise EvalError("c takes exactly 2 arguments", args) return CONS_COST, args.first().cons(args.rest().first()) -def op_first(args: SExp): +def op_first(args: _T_SExp) -> Tuple[int, _T_SExp]: if args.list_len() != 1: raise EvalError("f takes exactly 1 argument", args) return FIRST_COST, args.first().first() -def op_rest(args: SExp): +def op_rest(args: _T_SExp) -> Tuple[int, _T_SExp]: if args.list_len() != 1: raise EvalError("r takes exactly 1 argument", args) return REST_COST, args.first().rest() -def op_listp(args: SExp): +def op_listp(args: _T_SExp) -> Tuple[int, _T_SExp]: if args.list_len() != 1: raise EvalError("l takes exactly 1 argument", args) return LISTP_COST, args.true if args.first().listp() else args.false -def op_raise(args: SExp): +def op_raise(args: _T_SExp) -> Tuple[int, _T_SExp]: if args.list_len() == 1 and not args.first().listp(): raise EvalError("clvm raise", args.first()) else: raise EvalError("clvm raise", args) -def op_eq(args: SExp): +def op_eq(args: _T_SExp) -> Tuple[int, _T_SExp]: if args.list_len() != 2: raise EvalError("= takes exactly 2 arguments", args) a0 = args.first() diff --git a/clvm/operators.py b/clvm/operators.py index d20de8fb..4bdef192 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -168,25 +168,25 @@ def default_unknown_op(op: bytes, args: SExp) -> Tuple[int, SExp]: return (cost, SExp.null()) -class DefaultOperator(Protocol): +class OperatorProtocol(Protocol): def __call__(self, op: bytes, args: SExp) -> Tuple[int, SExp]: ... _T_OperatorDict = TypeVar("_T_OperatorDict", bound="OperatorDict") -class OperatorDict(dict): +class OperatorDict(Dict[bytes, OperatorProtocol]): """ This is a nice hack that adds `__call__` to a dictionary, so operators can be added dynamically. """ - unknown_op_handler: DefaultOperator + unknown_op_handler: OperatorProtocol quote_atom: int apply_atom: int # TODO: how do you create an instance if that requires passing in an instance? - def __new__(cls: Type[_T_OperatorDict], d: Dict, *args: object, **kwargs) -> _T_OperatorDict: + def __new__(cls: Type[_T_OperatorDict], d: Dict[bytes, OperatorProtocol], *args: object, **kwargs) -> _T_OperatorDict: """ `quote_atom` and `apply_atom` must be set `unknown_op_handler` has a default implementation diff --git a/clvm/serialize.py b/clvm/serialize.py index 59b856e1..aec94141 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -12,11 +12,16 @@ # If the first byte read is one of the following: # 1000 0000 -> 0 bytes : nil # 0000 0000 -> 1 byte : zero (b'\x00') +from __future__ import annotations + import io import typing from .CLVMObject import CLVMObject -from .SExp import SExp + + +if typing.TYPE_CHECKING: + from .SExp import SExp MAX_SINGLE_BYTE = 0x7F diff --git a/tests/operatordict_test.py b/tests/operatordict_test.py index 830dba5a..cee1bfe6 100644 --- a/tests/operatordict_test.py +++ b/tests/operatordict_test.py @@ -1,6 +1,8 @@ import unittest -from clvm.operators import OperatorDict +from typing import Dict + +from clvm.operators import OperatorProtocol, OperatorDict class OperatorDictTest(unittest.TestCase): @@ -9,7 +11,8 @@ def test_operatordict_constructor(self) -> None: either by object property or by keyword argument. Note that they cannot be specified in the operator dictionary itself. """ - d = {1: "hello", 2: "goodbye"} + # ignoring because apparently it doesn't matter for this test that the types are all wrong + d: Dict[bytes, OperatorProtocol] = {1: "hello", 2: "goodbye"} # type: ignore [dict-item] with self.assertRaises(AttributeError): o = OperatorDict(d) with self.assertRaises(AttributeError): diff --git a/tests/to_sexp_test.py b/tests/to_sexp_test.py index 00b997dc..968c1e8c 100644 --- a/tests/to_sexp_test.py +++ b/tests/to_sexp_test.py @@ -1,9 +1,9 @@ import unittest from dataclasses import dataclass -from typing import Optional, Tuple, Any +from typing import ClassVar, Optional, TYPE_CHECKING, Tuple, cast from clvm.SExp import SExp, looks_like_clvm_object, convert_atom_to_bytes -from clvm.CLVMObject import CLVMObject +from clvm.CLVMObject import CLVMObject, CLVMObjectLike def validate_sexp(sexp: SExp) -> None: @@ -16,7 +16,9 @@ def validate_sexp(sexp: SExp) -> None: v1, v2 = v.pair assert looks_like_clvm_object(v1) assert looks_like_clvm_object(v2) - s1, s2 = v.as_pair() + from_as_pair = v.as_pair() + assert from_as_pair is not None + s1, s2 = from_as_pair validate_stack.append(s1) validate_stack.append(s2) else: @@ -31,7 +33,9 @@ def print_leaves(tree: SExp) -> str: return "%d " % a[0] ret = "" - for i in tree.as_pair(): + from_as_pair = tree.as_pair() + assert from_as_pair is not None + for i in from_as_pair: ret += print_leaves(i) return ret @@ -45,7 +49,9 @@ def print_tree(tree: SExp) -> str: return "%d " % a[0] ret = "(" - for i in tree.as_pair(): + from_as_pair = tree.as_pair() + assert from_as_pair is not None + for i in from_as_pair: ret += print_tree(i) ret += ")" return ret @@ -93,6 +99,8 @@ def test_arbitrary_underlying_tree(self) -> None: # those types implement the CLVMObject protocol. This is an example of # a tree that's generated class GeneratedTree: + if TYPE_CHECKING: + _type_check: ClassVar[CLVMObjectLike] = cast("GeneratedTree", None) depth: int = 4 val: int = 0 @@ -108,13 +116,21 @@ def atom(self) -> Optional[bytes]: return None return bytes([self.val]) + @atom.setter + def atom(self, val: Optional[bytes]) -> None: + raise RuntimeError("setting not supported in this test class") + @property - def pair(self) -> Optional[Tuple[Any, Any]]: + def pair(self) -> Optional[Tuple[CLVMObjectLike, CLVMObjectLike]]: if self.depth == 0: return None new_depth: int = self.depth - 1 return (GeneratedTree(new_depth, self.val), GeneratedTree(new_depth, self.val + 2**new_depth)) + @pair.setter + def pair(self, val: Optional[Tuple[CLVMObjectLike, CLVMObjectLike]]) -> None: + raise RuntimeError("setting not supported in this test class") + tree = SExp.to(GeneratedTree(5, 0)) assert print_leaves(tree) == "0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 " + \ "16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 " @@ -161,7 +177,7 @@ def test_empty_list_conversions(self) -> None: def test_eager_conversion(self) -> None: with self.assertRaises(ValueError): - SExp.to(("foobar", (1, {}))) + SExp.to(("foobar", (1, {}))) # type: ignore[arg-type] def test_convert_atom(self) -> None: assert convert_atom_to_bytes(0x133742) == bytes([0x13, 0x37, 0x42]) @@ -181,7 +197,7 @@ def test_convert_atom(self) -> None: assert convert_atom_to_bytes([1, 2, 3]) with self.assertRaises(ValueError): - assert convert_atom_to_bytes((1, 2)) + assert convert_atom_to_bytes((1, 2)) # type: ignore[arg-type] with self.assertRaises(ValueError): - assert convert_atom_to_bytes({}) + assert convert_atom_to_bytes({}) # type: ignore[arg-type] From 78d02f5a4a855600d5513d67a2c3180e56be268c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Nov 2023 10:38:28 -0500 Subject: [PATCH 09/56] fixup --- tests/as_python_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/as_python_test.py b/tests/as_python_test.py index 06f4fff5..d1d1d3a0 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -151,7 +151,7 @@ def test_deep_recursion(self) -> None: self.assertEqual(from_as_pair[1].as_atom(), SExp.null()) v = from_as_pair[0] element = d[0] - assert isinstance(element, list) + assert isinstance(element, (list, bytes)) d = element self.assertEqual(v.as_atom(), b"2") @@ -187,15 +187,17 @@ def test_long_list(self) -> None: self.assertEqual(v.as_atom(), SExp.null()) def test_invalid_type(self) -> None: - s = SExp.to(dummy_class) # type: ignore[arg-type] with self.assertRaises(ValueError): + s = SExp.to(dummy_class) # type: ignore[arg-type] + # TODO: this note seems incorrect and neither of the following lines are run # conversions are deferred, this is where it will fail: b = list(s.as_iter()) print(b) def test_invalid_tuple(self) -> None: - s = SExp.to((dummy_class, dummy_class)) # type: ignore[arg-type] with self.assertRaises(ValueError): + s = SExp.to((dummy_class, dummy_class)) # type: ignore[arg-type] + # TODO: this note seems incorrect and neither of the following lines are run # conversions are deferred, this is where it will fail: b = list(s.as_iter()) print(b) From 89b1cfb5d061fbaed4dbc155196c082dbfd73523 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Nov 2023 15:46:45 -0500 Subject: [PATCH 10/56] more --- clvm/CLVMObject.py | 4 ++- clvm/SExp.py | 14 ++++---- clvm/as_python.py | 22 +++++++++---- clvm/core_ops.py | 6 ++-- clvm/more_ops.py | 18 +++++++---- clvm/operators.py | 65 +++++++++++++++++++++++++------------- clvm/run_program.py | 25 ++++++++------- clvm/serialize.py | 32 +++++++++++++++---- setup.py | 2 +- tests/operatordict_test.py | 16 +++++----- 10 files changed, 130 insertions(+), 74 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index d4dfcc04..e9ea3d2e 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -44,7 +44,9 @@ def __new__( raise ValueError("tuples must be of size 2, cannot create CLVMObject from: %s" % str(v)) self.pair = v self.atom = None - else: + elif isinstance(v, bytes): self.atom = v self.pair = None + else: + raise ValueError(f"cannot create CLVMObject from: {v!r}") return self diff --git a/clvm/SExp.py b/clvm/SExp.py index 7875b46e..c955b6f1 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -178,18 +178,18 @@ def as_bin(self) -> bytes: return f.getvalue() @classmethod - def to(class_, v: CastableType) -> "SExp": - if isinstance(v, class_): + def to(cls: typing.Type[_T_SExp], v: CastableType) -> _T_SExp: + if isinstance(v, cls): return v if looks_like_clvm_object(v): # TODO: maybe this can be done more cleanly - return class_(typing.cast(CLVMObjectLike, v)) + return cls(typing.cast(CLVMObjectLike, v)) # this will lazily convert elements - return class_(to_sexp_type(v)) + return cls(to_sexp_type(v)) - def cons(self: _T_SExp, right) -> _T_SExp: + def cons(self: _T_SExp, right: _T_SExp) -> _T_SExp: return self.to((self, right)) def first(self: _T_SExp) -> _T_SExp: @@ -214,9 +214,9 @@ def as_iter(self: _T_SExp) -> typing.Iterable[_T_SExp]: yield v.first() v = v.rest() - def __eq__(self, other: CastableType) -> bool: + def __eq__(self, other: object) -> bool: try: - other = self.to(other) + other = self.to(typing.cast(CastableType, other)) to_compare_stack = [(self, other)] while to_compare_stack: s1, s2 = to_compare_stack.pop() diff --git a/clvm/as_python.py b/clvm/as_python.py index 9370d21d..a72c9274 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -1,19 +1,27 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import Callable, List, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: from clvm.SExp import SExp +OpCallable = Callable[["OpStackType", "ValStackType"], None] + +ValStackType = List[SExp] +OpStackType = List[OpCallable] + +# TODO: hum... +PythonType = Union[int, bytes, str, List["PythonType"], Tuple["PythonType", "PythonType"]] + def as_python(sexp: SExp): - def _roll(op_stack, val_stack): + def _roll(op_stack: OpStackType, value_stack: ValStackType) -> None: v1 = val_stack.pop() v2 = val_stack.pop() val_stack.append(v1) val_stack.append(v2) - def _make_tuple(op_stack, val_stack): + def _make_tuple(op_stack: OpStackType, value_stack: ValStackType) -> None: left = val_stack.pop() right = val_stack.pop() if right == b"": @@ -24,7 +32,7 @@ def _make_tuple(op_stack, val_stack): else: val_stack.append((left, right)) - def _as_python(op_stack, val_stack): + def _as_python(op_stack: OpStackType, value_stack: ValStackType) -> None: t = val_stack.pop() pair = t.as_pair() if pair: @@ -36,10 +44,10 @@ def _as_python(op_stack, val_stack): val_stack.append(left) val_stack.append(right) else: - val_stack.append(t.as_atom()) + val_stack.append(t.atom) - op_stack = [_as_python] - val_stack = [sexp] + op_stack: OpStackType = [_as_python] + val_stack: ValStackType = [sexp] while op_stack: op_f = op_stack.pop() op_f(op_stack, val_stack) diff --git a/clvm/core_ops.py b/clvm/core_ops.py index 3a114125..d8a69c09 100644 --- a/clvm/core_ops.py +++ b/clvm/core_ops.py @@ -46,7 +46,7 @@ def op_rest(args: _T_SExp) -> Tuple[int, _T_SExp]: return REST_COST, args.first().rest() -def op_listp(args: _T_SExp) -> Tuple[int, _T_SExp]: +def op_listp(args: _T_SExp) -> Tuple[int, SExp]: if args.list_len() != 1: raise EvalError("l takes exactly 1 argument", args) return LISTP_COST, args.true if args.first().listp() else args.false @@ -59,7 +59,7 @@ def op_raise(args: _T_SExp) -> Tuple[int, _T_SExp]: raise EvalError("clvm raise", args) -def op_eq(args: _T_SExp) -> Tuple[int, _T_SExp]: +def op_eq(args: _T_SExp) -> Tuple[int, SExp]: if args.list_len() != 2: raise EvalError("= takes exactly 2 arguments", args) a0 = args.first() @@ -67,7 +67,9 @@ def op_eq(args: _T_SExp) -> Tuple[int, _T_SExp]: if a0.pair or a1.pair: raise EvalError("= on list", a0 if a0.pair else a1) b0 = a0.as_atom() + assert b0 is not None b1 = a1.as_atom() + assert b1 is not None cost = EQ_BASE_COST cost += (len(b0) + len(b1)) * EQ_COST_PER_BYTE return cost, (args.true if b0 == b1 else args.false) diff --git a/clvm/more_ops.py b/clvm/more_ops.py index 55690ba3..e6b48b7e 100644 --- a/clvm/more_ops.py +++ b/clvm/more_ops.py @@ -224,7 +224,7 @@ def op_gr_bytes(args: SExp) -> typing.Tuple[int, SExp]: return cost, args.true if b0 > b1 else args.false -def op_pubkey_for_exp(args: _T_SExp) -> typing.Tuple[_T_SExp, _T_SExp]: +def op_pubkey_for_exp(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: ((i0, l0),) = args_as_int_list("pubkey_for_exp", args, 1) i0 %= 0x73EDA753299D7D483339D80809A1D80553BDA402FFFE5BFEFFFFFFFF00000001 exponent = PrivateKey.from_bytes(i0.to_bytes(32, "big")) @@ -258,7 +258,8 @@ def op_strlen(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: a0 = args.first() if a0.pair: raise EvalError("strlen on list", a0) - size = len(a0.as_atom()) + assert a0.atom is not None + size = len(a0.atom) cost = STRLEN_BASE_COST + size * STRLEN_COST_PER_BYTE return malloc_cost(cost, args.to(size)) @@ -272,6 +273,7 @@ def op_substr(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: raise EvalError("substr on list", a0) s0 = a0.as_atom() + assert s0 is not None if arg_count == 2: i1, = list(args_as_int32("substr", args.rest())) @@ -292,7 +294,8 @@ def op_concat(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: for arg in args.as_iter(): if arg.pair: raise EvalError("concat on list", arg) - s.write(arg.as_atom()) + assert arg.atom is not None + s.write(arg.atom) cost += CONCAT_COST_PER_ARG r = s.getvalue() cost += len(r) * CONCAT_COST_PER_BYTE @@ -322,6 +325,7 @@ def op_lsh(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: raise EvalError("shift too large", args.to(i1)) # we actually want i0 to be an *unsigned* int a0 = args.first().as_atom() + assert a0 is not None i0 = int.from_bytes(a0, "big", signed=False) if i1 >= 0: r = i0 << i1 @@ -350,7 +354,7 @@ def binop_reduction( def op_logand(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: - def binop(a, b): + def binop(a: int, b: int) -> int: a &= b return a @@ -358,7 +362,7 @@ def binop(a, b): def op_logior(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: - def binop(a, b): + def binop(a: int, b: int) -> int: a |= b return a @@ -366,7 +370,7 @@ def binop(a, b): def op_logxor(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: - def binop(a, b): + def binop(a: int, b: int) -> int: a ^= b return a @@ -411,7 +415,7 @@ def op_all(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: return cost, args.to(r) -def op_softfork(args: SExp) -> typing.Tuple[int, bool]: +def op_softfork(args: SExp) -> typing.Tuple[int, SExp]: if args.list_len() < 1: raise EvalError("softfork takes at least 1 argument", args) a = args.first() diff --git a/clvm/operators.py b/clvm/operators.py index 4bdef192..c1ce92d9 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Dict, Tuple, Type, TypeVar +from typing import Dict, Iterator, Optional, Tuple, Type, TypeVar from typing_extensions import Protocol @@ -28,22 +28,16 @@ KEYWORDS = ( # core opcodes 0x01-x08 ". q a i c f r l x " - # opcodes on atoms as strings 0x09-0x0f "= >s sha256 substr strlen concat . " - # opcodes on atoms as ints 0x10-0x17 "+ - * / divmod > ash lsh " - # opcodes on atoms as vectors of bools 0x18-0x1c "logand logior logxor lognot . " - # opcodes for bls 1381 0x1d-0x1f "point_add pubkey_for_exp . " - # bool opcodes 0x20-0x23 "not any all . " - # misc 0x24 "softfork " ).split() @@ -68,11 +62,12 @@ } -def args_len(op_name, args: SExp): +def args_len(op_name: str, args: SExp) -> Iterator[int]: for arg in args.as_iter(): if arg.pair: raise EvalError("%s requires int args" % op_name, arg) - yield len(arg.as_atom()) + assert arg.atom is not None + yield len(arg.atom) # unknown ops are reserved if they start with 0xffff @@ -102,6 +97,7 @@ def args_len(op_name, args: SExp): # this means that unknown ops where cost_function is 1, 2, or 3, may still be # fatal errors if the arguments passed are not atoms. + def default_unknown_op(op: bytes, args: SExp) -> Tuple[int, SExp]: # any opcode starting with ffff is reserved (i.e. fatal error) # opcodes are not allowed to be empty @@ -158,6 +154,7 @@ def default_unknown_op(op: bytes, args: SExp) -> Tuple[int, SExp]: if arg.pair: raise EvalError("unknown op on list", arg) cost += CONCAT_COST_PER_ARG + assert arg.atom is not None length += len(arg.atom) cost += length * CONCAT_COST_PER_BYTE @@ -169,7 +166,13 @@ def default_unknown_op(op: bytes, args: SExp) -> Tuple[int, SExp]: class OperatorProtocol(Protocol): - def __call__(self, op: bytes, args: SExp) -> Tuple[int, SExp]: ... + def __call__(self, args: SExp) -> Tuple[int, SExp]: + ... + + +class UnknownOperatorProtocol(Protocol): + def __call__(self, op: bytes, args: SExp) -> Tuple[int, SExp]: + ... _T_OperatorDict = TypeVar("_T_OperatorDict", bound="OperatorDict") @@ -181,12 +184,21 @@ class OperatorDict(Dict[bytes, OperatorProtocol]): operators can be added dynamically. """ - unknown_op_handler: OperatorProtocol - quote_atom: int - apply_atom: int - - # TODO: how do you create an instance if that requires passing in an instance? - def __new__(cls: Type[_T_OperatorDict], d: Dict[bytes, OperatorProtocol], *args: object, **kwargs) -> _T_OperatorDict: + unknown_op_handler: UnknownOperatorProtocol + quote_atom: bytes + apply_atom: bytes + + # TODO: can we remove the args and kwargs? + # TODO: hint the overloads + def __new__( + cls: Type[_T_OperatorDict], + d: Dict[bytes, OperatorProtocol], + *args: object, + quote: Optional[bytes] = None, + apply: Optional[bytes] = None, + unknown_op_handler: UnknownOperatorProtocol = default_unknown_op, + **kwargs: object, + ) -> _T_OperatorDict: """ `quote_atom` and `apply_atom` must be set `unknown_op_handler` has a default implementation @@ -194,12 +206,19 @@ def __new__(cls: Type[_T_OperatorDict], d: Dict[bytes, OperatorProtocol], *args: We do not check if the opcode values for quote and apply exist in the passed-in dict """ self = super().__new__(cls, d) - self.quote_atom = kwargs["quote"] if "quote" in kwargs else d.quote_atom - self.apply_atom = kwargs["apply"] if "apply" in kwargs else d.apply_atom - if "unknown_op_handler" in kwargs: - self.unknown_op_handler = kwargs["unknown_op_handler"] + + if quote is None: + assert isinstance(d, OperatorDict) + self.quote_atom = d.quote_atom else: - self.unknown_op_handler = default_unknown_op + self.quote_atom = quote + + if apply is None: + assert isinstance(d, OperatorDict) + self.apply_atom = d.apply_atom + else: + self.apply_atom = apply + return self def __call__(self, op: bytes, arguments: SExp) -> Tuple[int, SExp]: @@ -214,6 +233,8 @@ def __call__(self, op: bytes, arguments: SExp) -> Tuple[int, SExp]: APPLY_ATOM = KEYWORD_TO_ATOM["a"] OPERATOR_LOOKUP = OperatorDict( - operators_for_module(KEYWORD_TO_ATOM, core_ops, OP_REWRITE), quote=QUOTE_ATOM, apply=APPLY_ATOM + operators_for_module(KEYWORD_TO_ATOM, core_ops, OP_REWRITE), + quote=QUOTE_ATOM, + apply=APPLY_ATOM, ) OPERATOR_LOOKUP.update(operators_for_module(KEYWORD_TO_ATOM, more_ops, OP_REWRITE)) diff --git a/clvm/run_program.py b/clvm/run_program.py index a8b5fac0..5db7a00b 100644 --- a/clvm/run_program.py +++ b/clvm/run_program.py @@ -1,8 +1,8 @@ -from typing import Any, Callable, List, Optional, Tuple +from typing import Callable, List, Optional, Tuple from .CLVMObject import CLVMObject from .EvalError import EvalError -from .SExp import SExp +from .SExp import CastableType, SExp from .operators import OperatorDict from .costs import ( @@ -13,16 +13,14 @@ PATH_LOOKUP_COST_PER_ZERO_BYTE ) -# the "Any" below should really be "OpStackType" but -# recursive types aren't supported by mypy - -OpCallable = Callable[[Any, "ValStackType"], int] +OpCallable = Callable[["OpStackType", "ValStackType"], int] +PreOpCallable = Callable[["OpStackType", "ValStackType"], None] ValStackType = List[SExp] OpStackType = List[OpCallable] -def to_pre_eval_op(pre_eval_f, to_sexp_f) -> Callable[[OpStackType, ValStackType], None]: +def to_pre_eval_op(pre_eval_f: Callable[[SExp, SExp], Optional[Callable[[SExp], object]]], to_sexp_f: Callable[[CastableType], SExp]) -> PreOpCallable: def my_pre_eval_op(op_stack: OpStackType, value_stack: ValStackType) -> None: v = to_sexp_f(value_stack[-1]) context = pre_eval_f(v.first(), v.rest()) @@ -39,7 +37,7 @@ def invoke_context_op( return my_pre_eval_op -def msb_mask(byte): +def msb_mask(byte: int) -> int: byte |= byte >> 1 byte |= byte >> 2 byte |= byte >> 4 @@ -48,14 +46,14 @@ def msb_mask(byte): def run_program( program: CLVMObject, - args: CLVMObject, + args: SExp, operator_lookup: OperatorDict, max_cost: Optional[int] = None, - pre_eval_f=None, + pre_eval_f: Optional[PreOpCallable] = None, ) -> Tuple[int, SExp]: _program = SExp.to(program) - if pre_eval_f: + if pre_eval_f is not None: pre_eval_op = to_pre_eval_op(pre_eval_f, _program.to) else: pre_eval_op = None @@ -129,7 +127,9 @@ def eval_op(op_stack: OpStackType, value_stack: ValStackType) -> int: operator = sexp.first() if operator.pair: - new_operator, must_be_nil = operator.as_pair() + from_as_pair = operator.as_pair() + assert from_as_pair is not None + new_operator, must_be_nil = from_as_pair if new_operator.pair or must_be_nil.atom != b"": raise EvalError("in ((X)...) syntax X must be lone atom", sexp) new_operand_list = sexp.rest() @@ -163,6 +163,7 @@ def apply_op(op_stack: OpStackType, value_stack: ValStackType) -> int: raise EvalError("internal error", operator) op = operator.as_atom() + assert op is not None if op == operator_lookup.apply_atom: if operand_list.list_len() != 2: raise EvalError("apply requires exactly 2 parameters", operand_list) diff --git a/clvm/serialize.py b/clvm/serialize.py index aec94141..04fb4a6d 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -31,6 +31,14 @@ T = typing.TypeVar("T") +OpCallable = typing.Callable[ + ["OpStackType", "ValStackType", typing.BinaryIO, typing.Type], None +] + +ValStackType = typing.List[SExp] +OpStackType = typing.List[OpCallable] + + def sexp_to_byte_iterator(sexp: SExp) -> typing.Iterator[bytes]: todo_stack = [sexp] while todo_stack: @@ -41,7 +49,8 @@ def sexp_to_byte_iterator(sexp: SExp) -> typing.Iterator[bytes]: todo_stack.append(pair[1]) todo_stack.append(pair[0]) else: - yield from atom_to_byte_iterator(sexp.as_atom()) + assert sexp.atom is not None + yield from atom_to_byte_iterator(sexp.atom) def atom_to_byte_iterator(as_atom: bytes) -> typing.Iterator[bytes]: @@ -90,7 +99,9 @@ def sexp_to_stream(sexp: SExp, f: typing.BinaryIO) -> None: f.write(b) -def _op_read_sexp(op_stack, val_stack, f: typing.BinaryIO, to_sexp) -> None: +def _op_read_sexp( + op_stack: OpStackType, val_stack: ValStackType, f: typing.BinaryIO, to_sexp: typing.Callable[[bytes], SExp], +) -> None: blob = f.read(1) if len(blob) == 0: raise ValueError("bad encoding") @@ -103,15 +114,20 @@ def _op_read_sexp(op_stack, val_stack, f: typing.BinaryIO, to_sexp) -> None: val_stack.append(_atom_from_stream(f, b, to_sexp)) -def _op_cons(op_stack, val_stack, f: typing.BinaryIO, to_sexp) -> None: +def _op_cons( + op_stack: OpStackType, + val_stack: ValStackType, + f: typing.BinaryIO, + to_sexp: typing.Callable[[typing.Tuple[SExp, SExp]], SExp], +) -> None: right = val_stack.pop() left = val_stack.pop() val_stack.append(to_sexp((left, right))) -def sexp_from_stream(f: typing.BinaryIO, to_sexp: typing.Callable[..., T]) -> T: - op_stack = [_op_read_sexp] - val_stack = [] +def sexp_from_stream(f: typing.BinaryIO, to_sexp: typing.Callable[[SExp], T]) -> T: + op_stack: OpStackType = [_op_read_sexp] + val_stack: ValStackType = [] while op_stack: func = op_stack.pop() @@ -171,7 +187,9 @@ def sexp_buffer_from_stream(f: typing.BinaryIO) -> bytes: return ret.getvalue() -def _atom_from_stream(f: typing.BinaryIO, b: int, to_sexp: typing.Callable[..., T]) -> T: +def _atom_from_stream( + f: typing.BinaryIO, b: int, to_sexp: typing.Callable[[bytes], T] +) -> T: if b == 0x80: return to_sexp(b"") if b <= MAX_SINGLE_BYTE: diff --git a/setup.py b/setup.py index 56118da1..3b5e0240 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ dependencies = [ "blspy>=0.9", - "typing-extensions~=4.0.0", # Backports of new typing module features + "typing-extensions~=4.0", # Backports of new typing module features ] dev_dependencies = [ diff --git a/tests/operatordict_test.py b/tests/operatordict_test.py index cee1bfe6..89a60950 100644 --- a/tests/operatordict_test.py +++ b/tests/operatordict_test.py @@ -12,21 +12,21 @@ def test_operatordict_constructor(self) -> None: Note that they cannot be specified in the operator dictionary itself. """ # ignoring because apparently it doesn't matter for this test that the types are all wrong - d: Dict[bytes, OperatorProtocol] = {1: "hello", 2: "goodbye"} # type: ignore [dict-item] + d: Dict[bytes, OperatorProtocol] = {b"\01": "hello", b"\02": "goodbye"} # type: ignore [dict-item] with self.assertRaises(AttributeError): o = OperatorDict(d) with self.assertRaises(AttributeError): - o = OperatorDict(d, apply=1) + o = OperatorDict(d, apply=b"\01") with self.assertRaises(AttributeError): - o = OperatorDict(d, quote=1) - o = OperatorDict(d, apply=1, quote=2) + o = OperatorDict(d, quote=b"\01") + o = OperatorDict(d, apply=b"\01", quote=b"\02") print(o) # Why does the constructed Operator dict contain entries for "apply":1 and "quote":2 ? # assert d == o - self.assertEqual(o.apply_atom, 1) - self.assertEqual(o.quote_atom, 2) + self.assertEqual(o.apply_atom, b"\01") + self.assertEqual(o.quote_atom, b"\02") # Test construction from an already existing OperatorDict o2 = OperatorDict(o) - self.assertEqual(o2.apply_atom, 1) - self.assertEqual(o2.quote_atom, 2) + self.assertEqual(o2.apply_atom, b"\01") + self.assertEqual(o2.quote_atom, b"\02") From 45a33a40c90b01ecb8e8e762434a2ba83088ac21 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 20 Nov 2023 20:48:31 -0500 Subject: [PATCH 11/56] py.typed --- clvm/py.typed | 0 setup.py | 3 +++ 2 files changed, 3 insertions(+) create mode 100644 clvm/py.typed diff --git a/clvm/py.typed b/clvm/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/setup.py b/setup.py index 3b5e0240..fc300973 100755 --- a/setup.py +++ b/setup.py @@ -41,4 +41,7 @@ "Bug Reports": "https://github.com/Chia-Network/clvm", "Source": "https://github.com/Chia-Network/clvm", }, + package_data={ + "": ["py.typed"], + }, ) From a54290923e233224dbeecaf665958d1435b0600f Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 20 Nov 2023 22:40:18 -0500 Subject: [PATCH 12/56] ignores... --- clvm/CLVMObject.py | 8 ++-- clvm/SExp.py | 93 +++++++++++++++++++++++++++++----------------- clvm/as_python.py | 78 +++++++++++++++++++++----------------- clvm/serialize.py | 7 ++-- 4 files changed, 111 insertions(+), 75 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index e9ea3d2e..f9ce4f78 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -13,8 +13,10 @@ class CLVMObjectLike(Protocol): pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] -_T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject") +PairType = typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] + +_T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject") class CLVMObject: """ @@ -26,7 +28,7 @@ class CLVMObject: # this is always a 2-tuple of an object implementing the CLVM object # protocol. - pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] + pair: PairType __slots__ = ["atom", "pair"] @staticmethod @@ -34,7 +36,7 @@ def __new__( class_: typing.Type[_T_CLVMObject], # TODO: which? review? # v: typing.Union[CLVMObject, CLVMObjectLike, typing.Tuple[CLVMObject, CLVMObject], bytes], - v: typing.Union[CLVMObjectLike, bytes], + v: typing.Union[CLVMObjectLike, bytes, PairType], ) -> _T_CLVMObject: if isinstance(v, class_): return v diff --git a/clvm/SExp.py b/clvm/SExp.py index c955b6f1..be286b8e 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -29,13 +29,15 @@ NULL = b"" -def looks_like_clvm_object(o: typing.Any) -> bool: +# TODO: can this be replaced by a runtime protocol check? +def looks_like_clvm_object(o: typing.Any) -> typing.TypeGuard[CLVMObjectLike]: d = dir(o) return "atom" in d and "pair" in d # this function recognizes some common types and turns them into plain bytes, def convert_atom_to_bytes( + # TODO: not any list, but an empty list v: typing.Union[bytes, str, int, None, typing.List, typing.SupportsBytes], ) -> bytes: @@ -55,25 +57,35 @@ def convert_atom_to_bytes( raise ValueError("can't cast %s (%s) to bytes" % (type(v), v)) +StackValType = typing.Union[CLVMObjectLike, typing.Tuple[CLVMObjectLike, CLVMObjectLike]] +StackType = typing.List[typing.Union[StackValType, "StackType"]] + # returns a clvm-object like object def to_sexp_type( - v: CastableType, + v: CLVMObjectLike, ) -> CLVMObjectLike: - stack = [v] - ops: typing.List[typing.Tuple[int, typing.Optional[int]]] = [(0, None)] # convert + stack: StackType = [v] + ops: typing.List[typing.Union[typing.Tuple[typing.Literal[0], None], typing.Tuple[int, int]]] = [(0, None)] # convert + + # TODO: can this type just be applied to the parameter instead? + internal_v: typing.Union[CLVMObjectLike, typing.Tuple, typing.List] + target: int + element: CLVMObjectLike while len(ops) > 0: - op, target = ops.pop() + op_target = ops.pop() # convert valuefrom .operators import OperatorDict - if op == 0: + # this form allows mypy to follow the not-none-ness of op_target[1] for all other operations + if op_target[1] is None: + assert op_target[0] == 0 if looks_like_clvm_object(stack[-1]): continue - v = stack.pop() - if isinstance(v, tuple): - if len(v) != 2: - raise ValueError("can't cast tuple of size %d" % len(v)) - left, right = v + internal_v = stack.pop() + if isinstance(internal_v, tuple): + if len(internal_v) != 2: + raise ValueError("can't cast tuple of size %d" % len(internal_v)) + left, right = internal_v target = len(stack) stack.append(CLVMObject((left, right))) if not looks_like_clvm_object(right): @@ -84,36 +96,47 @@ def to_sexp_type( stack.append(left) ops.append((1, target)) # set left ops.append((0, None)) # convert - continue - if isinstance(v, list): + elif isinstance(internal_v, list): target = len(stack) stack.append(CLVMObject(NULL)) - for _ in v: + for _ in internal_v: stack.append(_) ops.append((3, target)) # prepend list # we only need to convert if it's not already the right # type if not looks_like_clvm_object(_): ops.append((0, None)) # convert - continue - stack.append(CLVMObject(convert_atom_to_bytes(v))) - continue - - if op == 1: # set left - stack[target].pair = (CLVMObject(stack.pop()), stack[target].pair[1]) - continue - if op == 2: # set right - stack[target].pair = (stack[target].pair[0], CLVMObject(stack.pop())) - continue - if op == 3: # prepend list - stack[target] = CLVMObject((stack.pop(), stack[target])) - continue + else: + # TODO: do we have to ignore? + stack.append(CLVMObject(convert_atom_to_bytes(internal_v))) # type: ignore[arg-type] + elif op_target[0] == 1: # set left + target = op_target[1] + # TODO: do we have to ignore? + element = stack[target] # type: ignore[assignment] + pair = element.pair + assert pair is not None + # TODO: do we have to ignore? + element.pair = (CLVMObject(stack.pop()), pair[1]) # type: ignore[arg-type] + elif op_target[0] == 2: # set right + target = op_target[1] + # TODO: do we have to ignore? + element = stack[target] # type: ignore[assignment] + pair = element.pair + assert pair is not None + # TODO: do we have to ignore? + element.pair = (pair[0], CLVMObject(stack.pop())) # type: ignore[arg-type] + elif op_target[0] == 3: # prepend list + target = op_target[1] + # TODO: do we have to ignore? + stack[target] = CLVMObject((stack.pop(), stack[target])) # type: ignore[arg-type] + # TODO: what about an else to fail explicitly on an unknown op? # there's exactly one item left at this point if len(stack) != 1: raise ValueError("internal error") # stack[0] implements the clvm object protocol and can be wrapped by an SExp - return stack[0] + # TODO: do we have to ignore? + return stack[0] # type: ignore[return-value] _T_SExp = typing.TypeVar("_T_SExp", bound="SExp") @@ -134,9 +157,9 @@ class SExp: elements implementing the CLVM object protocol. Exactly one of "atom" and "pair" must be None. """ - true: "SExp" - false: "SExp" - __null__: "SExp" + true: typing.ClassVar["SExp"] + false: typing.ClassVar["SExp"] + __null__: typing.ClassVar["SExp"] # the underlying object implementing the clvm object protocol atom: typing.Optional[bytes] @@ -183,11 +206,11 @@ def to(cls: typing.Type[_T_SExp], v: CastableType) -> _T_SExp: return v if looks_like_clvm_object(v): - # TODO: maybe this can be done more cleanly - return cls(typing.cast(CLVMObjectLike, v)) + return cls(v) # this will lazily convert elements - return cls(to_sexp_type(v)) + # TODO: do we have to ignore? + return cls(to_sexp_type(v)) # type: ignore[arg-type] def cons(self: _T_SExp, right: _T_SExp) -> _T_SExp: return self.to((self, right)) @@ -242,7 +265,7 @@ def list_len(self) -> int: v = v.rest() return size - def as_python(self): + def as_python(self) -> typing.Any: return as_python(self) def __str__(self) -> str: diff --git a/clvm/as_python.py b/clvm/as_python.py index a72c9274..ee81d030 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -1,53 +1,63 @@ from __future__ import annotations -from typing import Callable, List, Tuple, TYPE_CHECKING, Union +from typing import Any, Callable, List, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: from clvm.SExp import SExp OpCallable = Callable[["OpStackType", "ValStackType"], None] -ValStackType = List[SExp] +ValType = Union[SExp, List["ValType"], Tuple["ValType", ...]] +ValStackType = List[Union[ValType, "ValStackType"]] OpStackType = List[OpCallable] # TODO: hum... PythonType = Union[int, bytes, str, List["PythonType"], Tuple["PythonType", "PythonType"]] -def as_python(sexp: SExp): - def _roll(op_stack: OpStackType, value_stack: ValStackType) -> None: - v1 = val_stack.pop() - v2 = val_stack.pop() - val_stack.append(v1) - val_stack.append(v2) - - def _make_tuple(op_stack: OpStackType, value_stack: ValStackType) -> None: - left = val_stack.pop() - right = val_stack.pop() - if right == b"": - val_stack.append([left]) - elif isinstance(right, list): - v = [left] + right - val_stack.append(v) - else: - val_stack.append((left, right)) - - def _as_python(op_stack: OpStackType, value_stack: ValStackType) -> None: - t = val_stack.pop() - pair = t.as_pair() - if pair: - left, right = pair - op_stack.append(_make_tuple) - op_stack.append(_as_python) - op_stack.append(_roll) - op_stack.append(_as_python) - val_stack.append(left) - val_stack.append(right) - else: - val_stack.append(t.atom) +def _roll(op_stack: OpStackType, val_stack: List[object]) -> None: + v1 = val_stack.pop() + v2 = val_stack.pop() + val_stack.append(v1) + val_stack.append(v2) + +# MakeTupleValType = Union[bytes, +MakeTupleValStackType = List[Union[bytes, Tuple[object, object], "MakeTupleValStackType"]] + + +def _make_tuple(op_stack: OpStackType, val_stack: MakeTupleValStackType) -> None: + left = val_stack.pop() + right = val_stack.pop() + if right == b"": + val_stack.append([left]) + elif isinstance(right, list): + v = [left] + right + val_stack.append(v) + else: + val_stack.append((left, right)) + + +def _as_python(op_stack: OpStackType, val_stack: List[SExp]) -> None: + t = val_stack.pop() + pair = t.as_pair() + if pair: + left, right = pair + op_stack.append(_make_tuple) + op_stack.append(_as_python) + op_stack.append(_roll) + op_stack.append(_as_python) + val_stack.append(left) + val_stack.append(right) + else: + # TODO: do we have to ignore? + val_stack.append(t.atom) # type:ignore[arg-type] + + +# TODO: probably have to just accept this Any, but... +def as_python(sexp: SExp) -> Any: op_stack: OpStackType = [_as_python] - val_stack: ValStackType = [sexp] + val_stack: List[SExp] = [sexp] while op_stack: op_f = op_stack.pop() op_f(op_stack, val_stack) diff --git a/clvm/serialize.py b/clvm/serialize.py index 77cb9af9..103e07a5 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -17,7 +17,7 @@ import io import typing -from .CLVMObject import CLVMObject +from .CLVMObject import CLVMObject, CLVMObjectLike if typing.TYPE_CHECKING: @@ -46,8 +46,9 @@ def sexp_to_byte_iterator(sexp: SExp) -> typing.Iterator[bytes]: pair = sexp.pair if pair: yield bytes([CONS_BOX_MARKER]) - todo_stack.append(pair[1]) - todo_stack.append(pair[0]) + # TODO: can we assume the pairs are necessarily SExp? can we hint this better so we require and know it + todo_stack.append(pair[1]) # type: ignore[arg-type] + todo_stack.append(pair[0]) # type: ignore[arg-type] else: assert sexp.atom is not None yield from atom_to_byte_iterator(sexp.atom) From 6abbfe9a785c4b5994306507d5bca4f1c559d53e Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 21 Nov 2023 10:18:40 -0500 Subject: [PATCH 13/56] stuff --- clvm/CLVMObject.py | 18 ++++++++++-------- clvm/as_python.py | 2 +- clvm/serialize.py | 2 +- tests/to_sexp_test.py | 3 ++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index f9ce4f78..26f65218 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -10,10 +10,10 @@ class CLVMObjectLike(Protocol): # restructuring all the classes. # TODO: can we make these read only? atom: typing.Optional[bytes] - pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] + pair: typing.Optional[PairType] -PairType = typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] +PairType = typing.Tuple[CLVMObjectLike, CLVMObjectLike] _T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject") @@ -28,7 +28,7 @@ class CLVMObject: # this is always a 2-tuple of an object implementing the CLVM object # protocol. - pair: PairType + pair: typing.Optional[PairType] __slots__ = ["atom", "pair"] @staticmethod @@ -36,7 +36,7 @@ def __new__( class_: typing.Type[_T_CLVMObject], # TODO: which? review? # v: typing.Union[CLVMObject, CLVMObjectLike, typing.Tuple[CLVMObject, CLVMObject], bytes], - v: typing.Union[CLVMObjectLike, bytes, PairType], + v: typing.Union[CLVMObject, bytes, PairType], ) -> _T_CLVMObject: if isinstance(v, class_): return v @@ -46,9 +46,11 @@ def __new__( raise ValueError("tuples must be of size 2, cannot create CLVMObject from: %s" % str(v)) self.pair = v self.atom = None - elif isinstance(v, bytes): - self.atom = v - self.pair = None + # TODO: discussing this + # elif isinstance(v, bytes): else: - raise ValueError(f"cannot create CLVMObject from: {v!r}") + self.atom = v # type: ignore[assignment] + self.pair = None + # else: + # raise ValueError(f"cannot create CLVMObject from: {v!r}") return self diff --git a/clvm/as_python.py b/clvm/as_python.py index ee81d030..2b89b4bf 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -7,7 +7,7 @@ OpCallable = Callable[["OpStackType", "ValStackType"], None] -ValType = Union[SExp, List["ValType"], Tuple["ValType", ...]] +ValType = Union["SExp", List["ValType"], Tuple["ValType", ...]] ValStackType = List[Union[ValType, "ValStackType"]] OpStackType = List[OpCallable] diff --git a/clvm/serialize.py b/clvm/serialize.py index 103e07a5..5abeb6ae 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -35,7 +35,7 @@ ["OpStackType", "ValStackType", typing.BinaryIO, typing.Type], None ] -ValStackType = typing.List[SExp] +ValStackType = typing.List["SExp"] OpStackType = typing.List[OpCallable] diff --git a/tests/to_sexp_test.py b/tests/to_sexp_test.py index 968c1e8c..588707fb 100644 --- a/tests/to_sexp_test.py +++ b/tests/to_sexp_test.py @@ -90,7 +90,8 @@ def test_wrap_sexp(self) -> None: # it's a bit of a layer violation that CLVMObject unwraps SExp, but we # rely on that in a fair number of places for now. We should probably # work towards phasing that out - o = CLVMObject(SExp.to(1)) + # TODO: yep, this is cheeating the system, discussing + o = CLVMObject(SExp.to(1)) # type: ignore[arg-type] assert o.atom == bytes([1]) def test_arbitrary_underlying_tree(self) -> None: From 1f51f7778906bb3fc1d57648a804b62d6081557a Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 21 Nov 2023 10:21:19 -0500 Subject: [PATCH 14/56] just ignore for now --- clvm/as_python.py | 18 ++++++++++++------ clvm/run_program.py | 3 ++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/clvm/as_python.py b/clvm/as_python.py index 2b89b4bf..525cdfc4 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -43,10 +43,14 @@ def _as_python(op_stack: OpStackType, val_stack: List[SExp]) -> None: pair = t.as_pair() if pair: left, right = pair - op_stack.append(_make_tuple) - op_stack.append(_as_python) - op_stack.append(_roll) - op_stack.append(_as_python) + # TODO: make this work, ignoring to look at other things + op_stack.append(_make_tuple) # type: ignore[arg-type] + # TODO: make this work, ignoring to look at other things + op_stack.append(_as_python) # type: ignore[arg-type] + # TODO: make this work, ignoring to look at other things + op_stack.append(_roll) # type: ignore[arg-type] + # TODO: make this work, ignoring to look at other things + op_stack.append(_as_python) # type: ignore[arg-type] val_stack.append(left) val_stack.append(right) else: @@ -56,9 +60,11 @@ def _as_python(op_stack: OpStackType, val_stack: List[SExp]) -> None: # TODO: probably have to just accept this Any, but... def as_python(sexp: SExp) -> Any: - op_stack: OpStackType = [_as_python] + # TODO: make this work, ignoring to look at other things + op_stack: OpStackType = [_as_python] # type: ignore[list-item] val_stack: List[SExp] = [sexp] while op_stack: op_f = op_stack.pop() - op_f(op_stack, val_stack) + # TODO: make this work, ignoring to look at other things + op_f(op_stack, val_stack) # type: ignore[arg-type] return val_stack[-1] diff --git a/clvm/run_program.py b/clvm/run_program.py index 5db7a00b..aa6d372b 100644 --- a/clvm/run_program.py +++ b/clvm/run_program.py @@ -54,7 +54,8 @@ def run_program( _program = SExp.to(program) if pre_eval_f is not None: - pre_eval_op = to_pre_eval_op(pre_eval_f, _program.to) + # TODO: make this work, ignoring to look at other things + pre_eval_op = to_pre_eval_op(pre_eval_f, _program.to) # type: ignore[arg-type] else: pre_eval_op = None From c9feb37bdcab70e4387df28fd75f869a84781c5c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 21 Nov 2023 10:23:45 -0500 Subject: [PATCH 15/56] note --- clvm/CLVMObject.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 26f65218..be7f18d8 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -8,7 +8,7 @@ class CLVMObjectLike(Protocol): # It's not clear if it is possible to express the exclusivity without maybe # restructuring all the classes. - # TODO: can we make these read only? + # TODO: can we make these read only? nope, to_sexp_type() depends on mutation atom: typing.Optional[bytes] pair: typing.Optional[PairType] @@ -18,6 +18,7 @@ class CLVMObjectLike(Protocol): _T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject") + class CLVMObject: """ This class implements the CLVM Object protocol in the simplest possible way, From a93ce62b847ab2e21c6daa0f8d59be2c82847e77 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 6 Dec 2023 21:05:00 -0500 Subject: [PATCH 16/56] tidy --- clvm/CLVMObject.py | 8 +++----- clvm/SExp.py | 6 +----- clvm/operators.py | 11 +++++++---- mypy.ini | 1 - setup.py | 1 - tests/as_python_test.py | 2 -- tests/to_sexp_test.py | 2 +- 7 files changed, 12 insertions(+), 19 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index be7f18d8..5cf50454 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -2,13 +2,11 @@ import typing -from typing_extensions import Protocol - -class CLVMObjectLike(Protocol): +class CLVMObjectLike(typing.Protocol): # It's not clear if it is possible to express the exclusivity without maybe - # restructuring all the classes. - # TODO: can we make these read only? nope, to_sexp_type() depends on mutation + # restructuring all the classes, such as having a separate instance for each + # of the atom and pair cases and hinting a union of a protocol of each type. atom: typing.Optional[bytes] pair: typing.Optional[PairType] diff --git a/clvm/SExp.py b/clvm/SExp.py index be286b8e..afda4e71 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -29,7 +29,6 @@ NULL = b"" -# TODO: can this be replaced by a runtime protocol check? def looks_like_clvm_object(o: typing.Any) -> typing.TypeGuard[CLVMObjectLike]: d = dir(o) return "atom" in d and "pair" in d @@ -37,7 +36,6 @@ def looks_like_clvm_object(o: typing.Any) -> typing.TypeGuard[CLVMObjectLike]: # this function recognizes some common types and turns them into plain bytes, def convert_atom_to_bytes( - # TODO: not any list, but an empty list v: typing.Union[bytes, str, int, None, typing.List, typing.SupportsBytes], ) -> bytes: @@ -67,16 +65,15 @@ def to_sexp_type( stack: StackType = [v] ops: typing.List[typing.Union[typing.Tuple[typing.Literal[0], None], typing.Tuple[int, int]]] = [(0, None)] # convert - # TODO: can this type just be applied to the parameter instead? internal_v: typing.Union[CLVMObjectLike, typing.Tuple, typing.List] target: int element: CLVMObjectLike while len(ops) > 0: op_target = ops.pop() - # convert valuefrom .operators import OperatorDict # this form allows mypy to follow the not-none-ness of op_target[1] for all other operations + # convert value if op_target[1] is None: assert op_target[0] == 0 if looks_like_clvm_object(stack[-1]): @@ -142,7 +139,6 @@ def to_sexp_type( _T_SExp = typing.TypeVar("_T_SExp", bound="SExp") -# TODO: Maybe there is some way to track atom vs. pair SExps to help hinting out a bit class SExp: """ SExp provides higher level API on top of any object implementing the CLVM diff --git a/clvm/operators.py b/clvm/operators.py index c1ce92d9..903a21ff 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,8 +1,6 @@ from __future__ import annotations -from typing import Dict, Iterator, Optional, Tuple, Type, TypeVar - -from typing_extensions import Protocol +from typing import Dict, Iterator, Optional, Protocol, Tuple, Type, TypeVar from . import core_ops, more_ops @@ -28,16 +26,22 @@ KEYWORDS = ( # core opcodes 0x01-x08 ". q a i c f r l x " + # opcodes on atoms as strings 0x09-0x0f "= >s sha256 substr strlen concat . " + # opcodes on atoms as ints 0x10-0x17 "+ - * / divmod > ash lsh " + # opcodes on atoms as vectors of bools 0x18-0x1c "logand logior logxor lognot . " + # opcodes for bls 1381 0x1d-0x1f "point_add pubkey_for_exp . " + # bool opcodes 0x20-0x23 "not any all . " + # misc 0x24 "softfork " ).split() @@ -97,7 +101,6 @@ def args_len(op_name: str, args: SExp) -> Iterator[int]: # this means that unknown ops where cost_function is 1, 2, or 3, may still be # fatal errors if the arguments passed are not atoms. - def default_unknown_op(op: bytes, args: SExp) -> Tuple[int, SExp]: # any opcode starting with ffff is reserved (i.e. fatal error) # opcodes are not allowed to be empty diff --git a/mypy.ini b/mypy.ini index 5ecdcae0..4e68bd74 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,5 @@ [mypy] files = clvm,tests,*.py -;ignore_missing_imports = True show_error_codes = True disallow_untyped_defs = True diff --git a/setup.py b/setup.py index fc300973..0bec79c7 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ dependencies = [ "blspy>=0.9", - "typing-extensions~=4.0", # Backports of new typing module features ] dev_dependencies = [ diff --git a/tests/as_python_test.py b/tests/as_python_test.py index d1d1d3a0..19f59ab6 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -189,7 +189,6 @@ def test_long_list(self) -> None: def test_invalid_type(self) -> None: with self.assertRaises(ValueError): s = SExp.to(dummy_class) # type: ignore[arg-type] - # TODO: this note seems incorrect and neither of the following lines are run # conversions are deferred, this is where it will fail: b = list(s.as_iter()) print(b) @@ -197,7 +196,6 @@ def test_invalid_type(self) -> None: def test_invalid_tuple(self) -> None: with self.assertRaises(ValueError): s = SExp.to((dummy_class, dummy_class)) # type: ignore[arg-type] - # TODO: this note seems incorrect and neither of the following lines are run # conversions are deferred, this is where it will fail: b = list(s.as_iter()) print(b) diff --git a/tests/to_sexp_test.py b/tests/to_sexp_test.py index 588707fb..fd82eff8 100644 --- a/tests/to_sexp_test.py +++ b/tests/to_sexp_test.py @@ -90,7 +90,7 @@ def test_wrap_sexp(self) -> None: # it's a bit of a layer violation that CLVMObject unwraps SExp, but we # rely on that in a fair number of places for now. We should probably # work towards phasing that out - # TODO: yep, this is cheeating the system, discussing + # TODO: yep, this is cheating the system, discussing o = CLVMObject(SExp.to(1)) # type: ignore[arg-type] assert o.atom == bytes([1]) From a377a48f37b271df35f2cca864765b4d48a417e1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 6 Dec 2023 21:08:13 -0500 Subject: [PATCH 17/56] fix --- clvm/operators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clvm/operators.py b/clvm/operators.py index 903a21ff..739895b1 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -222,6 +222,8 @@ def __new__( else: self.apply_atom = apply + self.unknown_op_handler = unknown_op_handler + return self def __call__(self, op: bytes, arguments: SExp) -> Tuple[int, SExp]: From baeb9dfc769357ad1f863927b561ad8f8806df1d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 6 Dec 2023 21:15:21 -0500 Subject: [PATCH 18/56] fix --- tests/operatordict_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/operatordict_test.py b/tests/operatordict_test.py index 89a60950..d6989361 100644 --- a/tests/operatordict_test.py +++ b/tests/operatordict_test.py @@ -13,11 +13,14 @@ def test_operatordict_constructor(self) -> None: """ # ignoring because apparently it doesn't matter for this test that the types are all wrong d: Dict[bytes, OperatorProtocol] = {b"\01": "hello", b"\02": "goodbye"} # type: ignore [dict-item] - with self.assertRaises(AttributeError): + # TODO: or do we want to retain the AttributeError behavior? + with self.assertRaises(AssertionError): o = OperatorDict(d) - with self.assertRaises(AttributeError): + # TODO: or do we want to retain the AttributeError behavior? + with self.assertRaises(AssertionError): o = OperatorDict(d, apply=b"\01") - with self.assertRaises(AttributeError): + # TODO: or do we want to retain the AttributeError behavior? + with self.assertRaises(AssertionError): o = OperatorDict(d, quote=b"\01") o = OperatorDict(d, apply=b"\01", quote=b"\02") print(o) From 01dac3010ea3a30734864fb5fdd519a05b0b96db Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 6 Dec 2023 21:27:05 -0500 Subject: [PATCH 19/56] flake8 --- clvm/SExp.py | 4 +++- clvm/op_utils.py | 6 +++++- clvm/run_program.py | 5 ++++- clvm/serialize.py | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index afda4e71..5cb489d5 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -58,12 +58,14 @@ def convert_atom_to_bytes( StackValType = typing.Union[CLVMObjectLike, typing.Tuple[CLVMObjectLike, CLVMObjectLike]] StackType = typing.List[typing.Union[StackValType, "StackType"]] + # returns a clvm-object like object def to_sexp_type( v: CLVMObjectLike, ) -> CLVMObjectLike: stack: StackType = [v] - ops: typing.List[typing.Union[typing.Tuple[typing.Literal[0], None], typing.Tuple[int, int]]] = [(0, None)] # convert + # convert + ops: typing.List[typing.Union[typing.Tuple[typing.Literal[0], None], typing.Tuple[int, int]]] = [(0, None)] internal_v: typing.Union[CLVMObjectLike, typing.Tuple, typing.List] target: int diff --git a/clvm/op_utils.py b/clvm/op_utils.py index ad3a20a0..0bbca458 100644 --- a/clvm/op_utils.py +++ b/clvm/op_utils.py @@ -2,7 +2,11 @@ from typing import Callable, Dict, Optional -def operators_for_dict(keyword_to_atom: Dict, op_dict: Dict[str, Callable], op_name_lookup: Optional[Dict] = None) -> Dict: +def operators_for_dict( + keyword_to_atom: Dict, + op_dict: Dict[str, Callable], + op_name_lookup: Optional[Dict] = None, +) -> Dict: if op_name_lookup is None: op_name_lookup = {} diff --git a/clvm/run_program.py b/clvm/run_program.py index aa6d372b..00014d14 100644 --- a/clvm/run_program.py +++ b/clvm/run_program.py @@ -20,7 +20,10 @@ OpStackType = List[OpCallable] -def to_pre_eval_op(pre_eval_f: Callable[[SExp, SExp], Optional[Callable[[SExp], object]]], to_sexp_f: Callable[[CastableType], SExp]) -> PreOpCallable: +def to_pre_eval_op( + pre_eval_f: Callable[[SExp, SExp], Optional[Callable[[SExp], object]]], + to_sexp_f: Callable[[CastableType], SExp], +) -> PreOpCallable: def my_pre_eval_op(op_stack: OpStackType, value_stack: ValStackType) -> None: v = to_sexp_f(value_stack[-1]) context = pre_eval_f(v.first(), v.rest()) diff --git a/clvm/serialize.py b/clvm/serialize.py index 5abeb6ae..b3709379 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -17,7 +17,7 @@ import io import typing -from .CLVMObject import CLVMObject, CLVMObjectLike +from .CLVMObject import CLVMObject if typing.TYPE_CHECKING: From 12f2e2e5b01a2d8fb5646a68c6e7ea3b136528a3 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 6 Dec 2023 21:33:12 -0500 Subject: [PATCH 20/56] typing_externsions.TypeGuard --- clvm/SExp.py | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index 5cb489d5..25595ff3 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -1,6 +1,7 @@ import io import typing +import typing_extensions from .as_python import as_python from .CLVMObject import CLVMObject, CLVMObjectLike @@ -29,7 +30,7 @@ NULL = b"" -def looks_like_clvm_object(o: typing.Any) -> typing.TypeGuard[CLVMObjectLike]: +def looks_like_clvm_object(o: typing.Any) -> typing_extensions.TypeGuard[CLVMObjectLike]: d = dir(o) return "atom" in d and "pair" in d diff --git a/setup.py b/setup.py index 0bec79c7..6bc87497 100755 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ dependencies = [ "blspy>=0.9", + "typing-extensions~=4.0", ] dev_dependencies = [ From ba0ea9adf2db4148259977a19b9426ab11e026fe Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 7 Dec 2023 11:01:45 -0500 Subject: [PATCH 21/56] Apply suggestions from code review Co-authored-by: Richard Kiss --- clvm/SExp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index 25595ff3..ac8188b7 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -23,7 +23,7 @@ int, None, list, - typing.Tuple["CastableType", ...], + typing.Tuple["CastableType", "CastableType"], ] @@ -37,7 +37,7 @@ def looks_like_clvm_object(o: typing.Any) -> typing_extensions.TypeGuard[CLVMObj # this function recognizes some common types and turns them into plain bytes, def convert_atom_to_bytes( - v: typing.Union[bytes, str, int, None, typing.List, typing.SupportsBytes], + v: typing.Union[bytes, str, int, None, typing.List[typing_extensions.Never], typing.SupportsBytes], ) -> bytes: if isinstance(v, bytes): From b5d6e23fa8fdac1af03ca25a694f8b1af6a57564 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 7 Dec 2023 11:04:13 -0500 Subject: [PATCH 22/56] add newly needed ignore --- tests/to_sexp_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/to_sexp_test.py b/tests/to_sexp_test.py index fd82eff8..8b48fb58 100644 --- a/tests/to_sexp_test.py +++ b/tests/to_sexp_test.py @@ -195,7 +195,7 @@ def test_convert_atom(self) -> None: assert convert_atom_to_bytes(DummyByteConvertible()) == b"foobar" with self.assertRaises(ValueError): - assert convert_atom_to_bytes([1, 2, 3]) + assert convert_atom_to_bytes([1, 2, 3]) # type: ignore[arg-type] with self.assertRaises(ValueError): assert convert_atom_to_bytes((1, 2)) # type: ignore[arg-type] From 7dee522039a83b1291abb0b7ade82599363a2468 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 7 Dec 2023 11:05:54 -0500 Subject: [PATCH 23/56] fix NestedTupleOfBytes --- tests/as_python_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/as_python_test.py b/tests/as_python_test.py index 19f59ab6..3e965d8e 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -26,7 +26,7 @@ def gen_tree(depth: int) -> SExp: H02 = fh("02") NestedListOfBytes = Union[bytes, List["NestedListOfBytes"]] -NestedTupleOfBytes = Union[bytes, Tuple["NestedTupleOfBytes", ...]] +NestedTupleOfBytes = Union[bytes, Tuple["NestedTupleOfBytes", "NestedTupleOfBytes"]] class AsPythonTest(unittest.TestCase): From 302813aa3530c0139cd3df085b225fae32df03f5 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 9 Dec 2023 13:24:32 -0500 Subject: [PATCH 24/56] `CLVMObjectLike` -> `CLVMStorage` --- clvm/CLVMObject.py | 6 +++--- clvm/SExp.py | 20 ++++++++++---------- tests/to_sexp_test.py | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 5cf50454..104c4b66 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -3,7 +3,7 @@ import typing -class CLVMObjectLike(typing.Protocol): +class CLVMStorage(typing.Protocol): # It's not clear if it is possible to express the exclusivity without maybe # restructuring all the classes, such as having a separate instance for each # of the atom and pair cases and hinting a union of a protocol of each type. @@ -11,7 +11,7 @@ class CLVMObjectLike(typing.Protocol): pair: typing.Optional[PairType] -PairType = typing.Tuple[CLVMObjectLike, CLVMObjectLike] +PairType = typing.Tuple[CLVMStorage, CLVMStorage] _T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject") @@ -34,7 +34,7 @@ class CLVMObject: def __new__( class_: typing.Type[_T_CLVMObject], # TODO: which? review? - # v: typing.Union[CLVMObject, CLVMObjectLike, typing.Tuple[CLVMObject, CLVMObject], bytes], + # v: typing.Union[CLVMObject, CLVMStorage, typing.Tuple[CLVMObject, CLVMObject], bytes], v: typing.Union[CLVMObject, bytes, PairType], ) -> _T_CLVMObject: if isinstance(v, class_): diff --git a/clvm/SExp.py b/clvm/SExp.py index ac8188b7..b1911f1b 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -4,7 +4,7 @@ import typing_extensions from .as_python import as_python -from .CLVMObject import CLVMObject, CLVMObjectLike +from .CLVMObject import CLVMObject, CLVMStorage from .EvalError import EvalError @@ -17,7 +17,7 @@ CastableType = typing.Union[ "SExp", - CLVMObjectLike, + CLVMStorage, bytes, str, int, @@ -30,7 +30,7 @@ NULL = b"" -def looks_like_clvm_object(o: typing.Any) -> typing_extensions.TypeGuard[CLVMObjectLike]: +def looks_like_clvm_object(o: typing.Any) -> typing_extensions.TypeGuard[CLVMStorage]: d = dir(o) return "atom" in d and "pair" in d @@ -56,21 +56,21 @@ def convert_atom_to_bytes( raise ValueError("can't cast %s (%s) to bytes" % (type(v), v)) -StackValType = typing.Union[CLVMObjectLike, typing.Tuple[CLVMObjectLike, CLVMObjectLike]] +StackValType = typing.Union[CLVMStorage, typing.Tuple[CLVMStorage, CLVMStorage]] StackType = typing.List[typing.Union[StackValType, "StackType"]] # returns a clvm-object like object def to_sexp_type( - v: CLVMObjectLike, -) -> CLVMObjectLike: + v: CLVMStorage, +) -> CLVMStorage: stack: StackType = [v] # convert ops: typing.List[typing.Union[typing.Tuple[typing.Literal[0], None], typing.Tuple[int, int]]] = [(0, None)] - internal_v: typing.Union[CLVMObjectLike, typing.Tuple, typing.List] + internal_v: typing.Union[CLVMStorage, typing.Tuple, typing.List] target: int - element: CLVMObjectLike + element: CLVMStorage while len(ops) > 0: op_target = ops.pop() @@ -165,9 +165,9 @@ class SExp: # this is a tuple of the otherlying CLVMObject-like objects. i.e. not # SExp objects with higher level functions, or None - pair: typing.Optional[typing.Tuple[CLVMObjectLike, CLVMObjectLike]] + pair: typing.Optional[typing.Tuple[CLVMStorage, CLVMStorage]] - def __init__(self, obj: CLVMObjectLike) -> None: + def __init__(self, obj: CLVMStorage) -> None: self.atom = obj.atom self.pair = obj.pair diff --git a/tests/to_sexp_test.py b/tests/to_sexp_test.py index e4e98b4f..8d471a82 100644 --- a/tests/to_sexp_test.py +++ b/tests/to_sexp_test.py @@ -3,7 +3,7 @@ from typing import ClassVar, Optional, TYPE_CHECKING, Tuple, cast from clvm.SExp import SExp, looks_like_clvm_object, convert_atom_to_bytes -from clvm.CLVMObject import CLVMObject, CLVMObjectLike +from clvm.CLVMObject import CLVMObject, CLVMStorage def validate_sexp(sexp: SExp) -> None: @@ -101,7 +101,7 @@ def test_arbitrary_underlying_tree(self) -> None: # a tree that's generated class GeneratedTree: if TYPE_CHECKING: - _type_check: ClassVar[CLVMObjectLike] = cast("GeneratedTree", None) + _type_check: ClassVar[CLVMStorage] = cast("GeneratedTree", None) depth: int = 4 val: int = 0 @@ -122,14 +122,14 @@ def atom(self, val: Optional[bytes]) -> None: raise RuntimeError("setting not supported in this test class") @property - def pair(self) -> Optional[Tuple[CLVMObjectLike, CLVMObjectLike]]: + def pair(self) -> Optional[Tuple[CLVMStorage, CLVMStorage]]: if self.depth == 0: return None new_depth: int = self.depth - 1 return (GeneratedTree(new_depth, self.val), GeneratedTree(new_depth, self.val + 2**new_depth)) @pair.setter - def pair(self, val: Optional[Tuple[CLVMObjectLike, CLVMObjectLike]]) -> None: + def pair(self, val: Optional[Tuple[CLVMStorage, CLVMStorage]]) -> None: raise RuntimeError("setting not supported in this test class") tree = SExp.to(GeneratedTree(5, 0)) From 502e5e19c49d0f4183c5213dd3c2884a26c7ffe0 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 9 Dec 2023 13:31:16 -0500 Subject: [PATCH 25/56] tidy hinting in `core_ops` --- clvm/core_ops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clvm/core_ops.py b/clvm/core_ops.py index d8a69c09..1a6ccf6d 100644 --- a/clvm/core_ops.py +++ b/clvm/core_ops.py @@ -1,4 +1,4 @@ -from typing import Tuple, TypeVar +from typing import Never, Tuple, TypeVar from .EvalError import EvalError from .SExp import SExp @@ -46,20 +46,20 @@ def op_rest(args: _T_SExp) -> Tuple[int, _T_SExp]: return REST_COST, args.first().rest() -def op_listp(args: _T_SExp) -> Tuple[int, SExp]: +def op_listp(args: SExp) -> Tuple[int, SExp]: if args.list_len() != 1: raise EvalError("l takes exactly 1 argument", args) return LISTP_COST, args.true if args.first().listp() else args.false -def op_raise(args: _T_SExp) -> Tuple[int, _T_SExp]: +def op_raise(args: SExp) -> Never: if args.list_len() == 1 and not args.first().listp(): raise EvalError("clvm raise", args.first()) else: raise EvalError("clvm raise", args) -def op_eq(args: _T_SExp) -> Tuple[int, SExp]: +def op_eq(args: SExp) -> Tuple[int, SExp]: if args.list_len() != 2: raise EvalError("= takes exactly 2 arguments", args) a0 = args.first() From 9422b0c21aa5cd5f079efdff84e6e6d52a581d02 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 9 Dec 2023 13:33:16 -0500 Subject: [PATCH 26/56] unused assignment in tests --- tests/as_python_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/as_python_test.py b/tests/as_python_test.py index db9b7bde..8a202f9b 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -188,11 +188,11 @@ def test_long_list(self) -> None: def test_invalid_type(self) -> None: with self.assertRaises(ValueError): - s = SExp.to(dummy_class) # type: ignore[arg-type] + SExp.to(dummy_class) # type: ignore[arg-type] def test_invalid_tuple(self) -> None: with self.assertRaises(ValueError): - s = SExp.to((dummy_class, dummy_class)) # type: ignore[arg-type] + SExp.to((dummy_class, dummy_class)) # type: ignore[arg-type] with self.assertRaises(ValueError): SExp.to((dummy_class, dummy_class, dummy_class)) # type: ignore[arg-type] From 29926ba37ee116e9368bbdd7a8eb4e39df12b1ad Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 9 Dec 2023 14:42:01 -0500 Subject: [PATCH 27/56] typing_extensions.Never --- clvm/core_ops.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clvm/core_ops.py b/clvm/core_ops.py index 1a6ccf6d..f27d0c6d 100644 --- a/clvm/core_ops.py +++ b/clvm/core_ops.py @@ -1,4 +1,6 @@ -from typing import Never, Tuple, TypeVar +from typing import Tuple, TypeVar + +from typing_extensions import Never from .EvalError import EvalError from .SExp import SExp From 12571a4e4e265ba779af5695bb32976ce01602e9 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 9 Dec 2023 22:24:16 -0500 Subject: [PATCH 28/56] tidy --- clvm/as_python.py | 4 ---- clvm/core_ops.py | 2 -- clvm/more_ops.py | 1 - clvm/operators.py | 27 +++++++++++++++++++++++++-- tests/operatordict_test.py | 6 +++--- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/clvm/as_python.py b/clvm/as_python.py index 525cdfc4..928875be 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -11,9 +11,6 @@ ValStackType = List[Union[ValType, "ValStackType"]] OpStackType = List[OpCallable] -# TODO: hum... -PythonType = Union[int, bytes, str, List["PythonType"], Tuple["PythonType", "PythonType"]] - def _roll(op_stack: OpStackType, val_stack: List[object]) -> None: v1 = val_stack.pop() @@ -58,7 +55,6 @@ def _as_python(op_stack: OpStackType, val_stack: List[SExp]) -> None: val_stack.append(t.atom) # type:ignore[arg-type] -# TODO: probably have to just accept this Any, but... def as_python(sexp: SExp) -> Any: # TODO: make this work, ignoring to look at other things op_stack: OpStackType = [_as_python] # type: ignore[list-item] diff --git a/clvm/core_ops.py b/clvm/core_ops.py index f27d0c6d..437fc435 100644 --- a/clvm/core_ops.py +++ b/clvm/core_ops.py @@ -18,8 +18,6 @@ _T_SExp = TypeVar("_T_SExp", bound=SExp) -# TODO: should all the int return types be more specific integer types? - def op_if(args: _T_SExp) -> Tuple[int, _T_SExp]: if args.list_len() != 3: diff --git a/clvm/more_ops.py b/clvm/more_ops.py index e6b48b7e..fadebd78 100644 --- a/clvm/more_ops.py +++ b/clvm/more_ops.py @@ -75,7 +75,6 @@ def op_sha256(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: return malloc_cost(cost, args.to(h.digest())) -# TODO: can we get more specific about op_name such as with a typing.Literal? def args_as_ints(op_name: str, args: SExp) -> typing.Iterator[typing.Tuple[int, int]]: for arg in args.as_iter(): if arg.atom is None: diff --git a/clvm/operators.py b/clvm/operators.py index 739895b1..bf1ab45d 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Dict, Iterator, Optional, Protocol, Tuple, Type, TypeVar +from typing import Dict, Iterator, Literal, Optional, Protocol, Tuple, Type, TypeVar, overload from . import core_ops, more_ops @@ -192,7 +192,30 @@ class OperatorDict(Dict[bytes, OperatorProtocol]): apply_atom: bytes # TODO: can we remove the args and kwargs? - # TODO: hint the overloads + @overload + def __new__( + cls: Type[_T_OperatorDict], + d: Dict[bytes, OperatorProtocol], + *args: object, + quote: bytes, + apply: bytes, + unknown_op_handler: UnknownOperatorProtocol = default_unknown_op, + **kwargs: object, + ) -> _T_OperatorDict: + ... + + @overload + def __new__( + cls: Type[_T_OperatorDict], + d: OperatorDict, + *args: object, + quote: Optional[bytes] = None, + apply: Optional[bytes] = None, + unknown_op_handler: UnknownOperatorProtocol = default_unknown_op, + **kwargs: object, + ) -> _T_OperatorDict: + ... + def __new__( cls: Type[_T_OperatorDict], d: Dict[bytes, OperatorProtocol], diff --git a/tests/operatordict_test.py b/tests/operatordict_test.py index 1cfca459..a6315ab7 100644 --- a/tests/operatordict_test.py +++ b/tests/operatordict_test.py @@ -15,13 +15,13 @@ def test_operatordict_constructor(self) -> None: d: Dict[bytes, OperatorProtocol] = {b"\01": "hello", b"\02": "goodbye"} # type: ignore [dict-item] # TODO: or do we want to retain the AttributeError behavior? with self.assertRaises(AssertionError): - OperatorDict(d) + OperatorDict(d) # type: ignore[call-overload] # TODO: or do we want to retain the AttributeError behavior? with self.assertRaises(AssertionError): - OperatorDict(d, apply=b"\01") + OperatorDict(d, apply=b"\01") # type: ignore[call-overload] # TODO: or do we want to retain the AttributeError behavior? with self.assertRaises(AssertionError): - OperatorDict(d, quote=b"\01") + OperatorDict(d, quote=b"\01") # type: ignore[call-overload] o = OperatorDict(d, apply=b"\01", quote=b"\02") print(o) # Why does the constructed Operator dict contain entries for "apply":1 and "quote":2 ? From 90fb89c5d780ef7b7d2aa4110266bd0c4b46c93b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 9 Dec 2023 22:28:51 -0500 Subject: [PATCH 29/56] tidy --- clvm/operators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clvm/operators.py b/clvm/operators.py index bf1ab45d..97ff4d72 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Dict, Iterator, Literal, Optional, Protocol, Tuple, Type, TypeVar, overload +from typing import Dict, Iterator, Optional, Protocol, Tuple, Type, TypeVar, overload from . import core_ops, more_ops From 55a514d0573d391fb2d9fe0dea52196c9769581e Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 13 Dec 2023 19:09:09 -0500 Subject: [PATCH 30/56] more tidy --- clvm/CLVMObject.py | 21 +++++++++------------ clvm/as_python.py | 44 ++++++++++++++++++++------------------------ clvm/operators.py | 4 ++-- 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 104c4b66..04cffdf0 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -33,23 +33,20 @@ class CLVMObject: @staticmethod def __new__( class_: typing.Type[_T_CLVMObject], - # TODO: which? review? - # v: typing.Union[CLVMObject, CLVMStorage, typing.Tuple[CLVMObject, CLVMObject], bytes], - v: typing.Union[CLVMObject, bytes, PairType], + v: typing.Union[_T_CLVMObject, bytes, PairType], ) -> _T_CLVMObject: if isinstance(v, class_): return v + # mypy does not realize that the isinstance check is type narrowing like this + narrowed_v: typing.Union[bytes, PairType] = v # type: ignore[assignment] + self = super(CLVMObject, class_).__new__(class_) - if isinstance(v, tuple): - if len(v) != 2: - raise ValueError("tuples must be of size 2, cannot create CLVMObject from: %s" % str(v)) - self.pair = v + if isinstance(narrowed_v, tuple): + if len(narrowed_v) != 2: + raise ValueError("tuples must be of size 2, cannot create CLVMObject from: %s" % str(narrowed_v)) + self.pair = narrowed_v self.atom = None - # TODO: discussing this - # elif isinstance(v, bytes): else: - self.atom = v # type: ignore[assignment] + self.atom = narrowed_v self.pair = None - # else: - # raise ValueError(f"cannot create CLVMObject from: {v!r}") return self diff --git a/clvm/as_python.py b/clvm/as_python.py index 928875be..c100099d 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -1,31 +1,33 @@ from __future__ import annotations -from typing import Any, Callable, List, Tuple, TYPE_CHECKING, Union +from typing import Any, Callable, List, Tuple, TYPE_CHECKING, Union, cast if TYPE_CHECKING: from clvm.SExp import SExp OpCallable = Callable[["OpStackType", "ValStackType"], None] -ValType = Union["SExp", List["ValType"], Tuple["ValType", ...]] -ValStackType = List[Union[ValType, "ValStackType"]] +PythonReturnType = Union[bytes, Tuple["PythonReturnType", "PythonReturnType"], List["PythonReturnType"]] + +ValType = Union["SExp", PythonReturnType] +ValStackType = List[ValType] + OpStackType = List[OpCallable] -def _roll(op_stack: OpStackType, val_stack: List[object]) -> None: +def _roll(op_stack: OpStackType, val_stack: ValStackType) -> None: v1 = val_stack.pop() v2 = val_stack.pop() val_stack.append(v1) val_stack.append(v2) -# MakeTupleValType = Union[bytes, MakeTupleValStackType = List[Union[bytes, Tuple[object, object], "MakeTupleValStackType"]] -def _make_tuple(op_stack: OpStackType, val_stack: MakeTupleValStackType) -> None: - left = val_stack.pop() - right = val_stack.pop() +def _make_tuple(op_stack: OpStackType, val_stack: ValStackType) -> None: + left: PythonReturnType = val_stack.pop() # type: ignore[assignment] + right: PythonReturnType = val_stack.pop() # type: ignore[assignment] if right == b"": val_stack.append([left]) elif isinstance(right, list): @@ -35,32 +37,26 @@ def _make_tuple(op_stack: OpStackType, val_stack: MakeTupleValStackType) -> None val_stack.append((left, right)) -def _as_python(op_stack: OpStackType, val_stack: List[SExp]) -> None: - t = val_stack.pop() +def _as_python(op_stack: OpStackType, val_stack: ValStackType) -> None: + t: SExp = val_stack.pop() # type: ignore[assignment] pair = t.as_pair() if pair: left, right = pair - # TODO: make this work, ignoring to look at other things - op_stack.append(_make_tuple) # type: ignore[arg-type] - # TODO: make this work, ignoring to look at other things - op_stack.append(_as_python) # type: ignore[arg-type] - # TODO: make this work, ignoring to look at other things - op_stack.append(_roll) # type: ignore[arg-type] - # TODO: make this work, ignoring to look at other things - op_stack.append(_as_python) # type: ignore[arg-type] + op_stack.append(_make_tuple) + op_stack.append(_as_python) + op_stack.append(_roll) + op_stack.append(_as_python) val_stack.append(left) val_stack.append(right) else: - # TODO: do we have to ignore? + # we know that t.atom is not None here because the pair is None val_stack.append(t.atom) # type:ignore[arg-type] def as_python(sexp: SExp) -> Any: - # TODO: make this work, ignoring to look at other things - op_stack: OpStackType = [_as_python] # type: ignore[list-item] - val_stack: List[SExp] = [sexp] + op_stack: OpStackType = [_as_python] + val_stack: ValStackType = [sexp] while op_stack: op_f = op_stack.pop() - # TODO: make this work, ignoring to look at other things - op_f(op_stack, val_stack) # type: ignore[arg-type] + op_f(op_stack, val_stack) return val_stack[-1] diff --git a/clvm/operators.py b/clvm/operators.py index 97ff4d72..d76d4042 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -70,8 +70,8 @@ def args_len(op_name: str, args: SExp) -> Iterator[int]: for arg in args.as_iter(): if arg.pair: raise EvalError("%s requires int args" % op_name, arg) - assert arg.atom is not None - yield len(arg.atom) + # not a pair so must be an atom + yield len(arg.atom) # type: ignore[arg-type] # unknown ops are reserved if they start with 0xffff From 142ecd17559fa528c30413d8f289aa473ef1f593 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 13 Dec 2023 19:16:39 -0500 Subject: [PATCH 31/56] oops --- clvm/as_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clvm/as_python.py b/clvm/as_python.py index c100099d..41ecf9a9 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Callable, List, Tuple, TYPE_CHECKING, Union, cast +from typing import Any, Callable, List, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: from clvm.SExp import SExp From ae671395c04f375e22a01d4483f0411ec5d4f9c3 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 13 Dec 2023 19:53:53 -0500 Subject: [PATCH 32/56] update note --- tests/to_sexp_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/to_sexp_test.py b/tests/to_sexp_test.py index 8d471a82..06b6dbca 100644 --- a/tests/to_sexp_test.py +++ b/tests/to_sexp_test.py @@ -90,7 +90,8 @@ def test_wrap_sexp(self) -> None: # it's a bit of a layer violation that CLVMObject unwraps SExp, but we # rely on that in a fair number of places for now. We should probably # work towards phasing that out - # TODO: yep, this is cheating the system, discussing + + # making sure this works despite hinting against it at CLVMObject o = CLVMObject(SExp.to(1)) # type: ignore[arg-type] assert o.atom == bytes([1]) From c76e7d9edbe36cc82cbae057a44a77469a0ac432 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 13 Dec 2023 20:13:28 -0500 Subject: [PATCH 33/56] shift around --- clvm/SExp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index b1911f1b..eed3307f 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -62,9 +62,10 @@ def convert_atom_to_bytes( # returns a clvm-object like object def to_sexp_type( - v: CLVMStorage, + v: CastableType, ) -> CLVMStorage: - stack: StackType = [v] + # TODO: this all needs reviewed + stack: StackType = [v] # type: ignore[list-item] # convert ops: typing.List[typing.Union[typing.Tuple[typing.Literal[0], None], typing.Tuple[int, int]]] = [(0, None)] @@ -208,8 +209,7 @@ def to(cls: typing.Type[_T_SExp], v: CastableType) -> _T_SExp: return cls(v) # this will lazily convert elements - # TODO: do we have to ignore? - return cls(to_sexp_type(v)) # type: ignore[arg-type] + return cls(to_sexp_type(v)) def cons(self: _T_SExp, right: _T_SExp) -> _T_SExp: return self.to((self, right)) From 393e1ba9e7a02124f77b969fec2ecb78e66a78ed Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 14:57:11 -0500 Subject: [PATCH 34/56] more --- clvm/CLVMObject.py | 3 +++ clvm/SExp.py | 29 +++++++++++++++++------------ clvm/more_ops.py | 18 ++++-------------- clvm/operators.py | 7 ------- clvm/run_program.py | 8 ++++---- clvm/serialize.py | 9 ++++----- 6 files changed, 32 insertions(+), 42 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index 04cffdf0..eb80aa46 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -16,6 +16,8 @@ class CLVMStorage(typing.Protocol): _T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject") +if typing.TYPE_CHECKING: + from .SExp import CastableType class CLVMObject: """ @@ -27,6 +29,7 @@ class CLVMObject: # this is always a 2-tuple of an object implementing the CLVM object # protocol. + # pair: typing.Optional[typing.Tuple[CastableType, CastableType]] pair: typing.Optional[PairType] __slots__ = ["atom", "pair"] diff --git a/clvm/SExp.py b/clvm/SExp.py index eed3307f..65f038af 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -56,20 +56,23 @@ def convert_atom_to_bytes( raise ValueError("can't cast %s (%s) to bytes" % (type(v), v)) -StackValType = typing.Union[CLVMStorage, typing.Tuple[CLVMStorage, CLVMStorage]] -StackType = typing.List[typing.Union[StackValType, "StackType"]] +ValType = typing.Union["SExp", CastableType] +StackType = typing.List[ValType] + +# StackValType = typing.Union[CLVMStorage, typing.Tuple[CLVMStorage, CLVMStorage]] +# StackType = typing.List[typing.Union[StackValType, "StackType"]] # returns a clvm-object like object +@typing.no_type_check def to_sexp_type( v: CastableType, ) -> CLVMStorage: - # TODO: this all needs reviewed - stack: StackType = [v] # type: ignore[list-item] + stack: StackType = [v] # convert ops: typing.List[typing.Union[typing.Tuple[typing.Literal[0], None], typing.Tuple[int, int]]] = [(0, None)] - internal_v: typing.Union[CLVMStorage, typing.Tuple, typing.List] + internal_v: CastableType #typing.Union[CLVMStorage, typing.Tuple, typing.List] target: int element: CLVMStorage @@ -88,6 +91,8 @@ def to_sexp_type( raise ValueError("can't cast tuple of size %d" % len(internal_v)) left, right = internal_v target = len(stack) + # if not looks_like_clvm_object(left) or not looks_like_clvm_object(right): + # assert False stack.append(CLVMObject((left, right))) if not looks_like_clvm_object(right): stack.append(right) @@ -109,27 +114,27 @@ def to_sexp_type( ops.append((0, None)) # convert else: # TODO: do we have to ignore? - stack.append(CLVMObject(convert_atom_to_bytes(internal_v))) # type: ignore[arg-type] + stack.append(CLVMObject(convert_atom_to_bytes(internal_v))) # taype: ignore[arg-type] elif op_target[0] == 1: # set left target = op_target[1] # TODO: do we have to ignore? - element = stack[target] # type: ignore[assignment] + element = stack[target] # taype: ignore[assignment] pair = element.pair assert pair is not None # TODO: do we have to ignore? - element.pair = (CLVMObject(stack.pop()), pair[1]) # type: ignore[arg-type] + element.pair = (CLVMObject(stack.pop()), pair[1]) # taype: ignore[arg-type] elif op_target[0] == 2: # set right target = op_target[1] # TODO: do we have to ignore? - element = stack[target] # type: ignore[assignment] + element = stack[target] # taype: ignore[assignment] pair = element.pair assert pair is not None # TODO: do we have to ignore? - element.pair = (pair[0], CLVMObject(stack.pop())) # type: ignore[arg-type] + element.pair = (pair[0], CLVMObject(stack.pop())) # taype: ignore[arg-type] elif op_target[0] == 3: # prepend list target = op_target[1] # TODO: do we have to ignore? - stack[target] = CLVMObject((stack.pop(), stack[target])) # type: ignore[arg-type] + stack[target] = CLVMObject((stack.pop(), stack[target])) # taype: ignore[arg-type] # TODO: what about an else to fail explicitly on an unknown op? # there's exactly one item left at this point if len(stack) != 1: @@ -137,7 +142,7 @@ def to_sexp_type( # stack[0] implements the clvm object protocol and can be wrapped by an SExp # TODO: do we have to ignore? - return stack[0] # type: ignore[return-value] + return stack[0] # taype: ignore[return-value] _T_SExp = typing.TypeVar("_T_SExp", bound="SExp") diff --git a/clvm/more_ops.py b/clvm/more_ops.py index fadebd78..71d36988 100644 --- a/clvm/more_ops.py +++ b/clvm/more_ops.py @@ -168,20 +168,10 @@ def op_divmod(args: _T_SExp) -> typing.Tuple[int, _T_SExp]: raise EvalError("divmod with 0", args.to(i0)) cost += (l0 + l1) * DIVMOD_COST_PER_BYTE q, r = divmod(i0, i1) - q1 = args.to(q) - r1 = args.to(r) - if q1.atom is None: - # TODO: Should this (and other added TypeErrors) be EvalErrors? Using - # TypeErrors because that matches the type of the exception that would - # have been thrown in the existing code. This could also be done with - # something like below to avoid any runtime differences, but it is also - # a bit ugly. - # - # q1_atom: int = args.to(q).atom # type: ignore[assignment] - raise TypeError(f"Internal error, quotient must be an atom, got: {q1}") - if r1.atom is None: - raise TypeError(f"Internal error, quotient must be an atom, got: {r1}") - cost += (len(q1.atom) + len(r1.atom)) * MALLOC_COST_PER_BYTE + # since q and r are integers, the atoms will be non-None + q1_atom: bytes = args.to(q).atom # type: ignore[assignment] + r1_atom: bytes = args.to(r).atom # type: ignore[assignment] + cost += (len(q1_atom) + len(r1_atom)) * MALLOC_COST_PER_BYTE return cost, args.to((q, r)) diff --git a/clvm/operators.py b/clvm/operators.py index d76d4042..4b2d834b 100644 --- a/clvm/operators.py +++ b/clvm/operators.py @@ -191,16 +191,13 @@ class OperatorDict(Dict[bytes, OperatorProtocol]): quote_atom: bytes apply_atom: bytes - # TODO: can we remove the args and kwargs? @overload def __new__( cls: Type[_T_OperatorDict], d: Dict[bytes, OperatorProtocol], - *args: object, quote: bytes, apply: bytes, unknown_op_handler: UnknownOperatorProtocol = default_unknown_op, - **kwargs: object, ) -> _T_OperatorDict: ... @@ -208,22 +205,18 @@ def __new__( def __new__( cls: Type[_T_OperatorDict], d: OperatorDict, - *args: object, quote: Optional[bytes] = None, apply: Optional[bytes] = None, unknown_op_handler: UnknownOperatorProtocol = default_unknown_op, - **kwargs: object, ) -> _T_OperatorDict: ... def __new__( cls: Type[_T_OperatorDict], d: Dict[bytes, OperatorProtocol], - *args: object, quote: Optional[bytes] = None, apply: Optional[bytes] = None, unknown_op_handler: UnknownOperatorProtocol = default_unknown_op, - **kwargs: object, ) -> _T_OperatorDict: """ `quote_atom` and `apply_atom` must be set diff --git a/clvm/run_program.py b/clvm/run_program.py index 00014d14..85529add 100644 --- a/clvm/run_program.py +++ b/clvm/run_program.py @@ -15,13 +15,14 @@ OpCallable = Callable[["OpStackType", "ValStackType"], int] PreOpCallable = Callable[["OpStackType", "ValStackType"], None] +PreEvalFunction = Callable[[SExp, SExp], Optional[Callable[[SExp], object]]] ValStackType = List[SExp] OpStackType = List[OpCallable] def to_pre_eval_op( - pre_eval_f: Callable[[SExp, SExp], Optional[Callable[[SExp], object]]], + pre_eval_f: PreEvalFunction, to_sexp_f: Callable[[CastableType], SExp], ) -> PreOpCallable: def my_pre_eval_op(op_stack: OpStackType, value_stack: ValStackType) -> None: @@ -52,13 +53,12 @@ def run_program( args: SExp, operator_lookup: OperatorDict, max_cost: Optional[int] = None, - pre_eval_f: Optional[PreOpCallable] = None, + pre_eval_f: Optional[PreEvalFunction] = None, ) -> Tuple[int, SExp]: _program = SExp.to(program) if pre_eval_f is not None: - # TODO: make this work, ignoring to look at other things - pre_eval_op = to_pre_eval_op(pre_eval_f, _program.to) # type: ignore[arg-type] + pre_eval_op = to_pre_eval_op(pre_eval_f, _program.to) else: pre_eval_op = None diff --git a/clvm/serialize.py b/clvm/serialize.py index b3709379..e60c09cd 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -17,7 +17,7 @@ import io import typing -from .CLVMObject import CLVMObject +from .CLVMObject import CLVMObject, CLVMStorage if typing.TYPE_CHECKING: @@ -39,16 +39,15 @@ OpStackType = typing.List[OpCallable] -def sexp_to_byte_iterator(sexp: SExp) -> typing.Iterator[bytes]: +def sexp_to_byte_iterator(sexp: CLVMStorage) -> typing.Iterator[bytes]: todo_stack = [sexp] while todo_stack: sexp = todo_stack.pop() pair = sexp.pair if pair: yield bytes([CONS_BOX_MARKER]) - # TODO: can we assume the pairs are necessarily SExp? can we hint this better so we require and know it - todo_stack.append(pair[1]) # type: ignore[arg-type] - todo_stack.append(pair[0]) # type: ignore[arg-type] + todo_stack.append(pair[1]) + todo_stack.append(pair[0]) else: assert sexp.atom is not None yield from atom_to_byte_iterator(sexp.atom) From d77164db844a60e708af266cd4413a7118831e6d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 14:59:37 -0500 Subject: [PATCH 35/56] undo --- clvm/SExp.py | 73 +++++++++++++++++++--------------------------------- 1 file changed, 26 insertions(+), 47 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index 65f038af..f1518b1d 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -68,31 +68,21 @@ def convert_atom_to_bytes( def to_sexp_type( v: CastableType, ) -> CLVMStorage: - stack: StackType = [v] - # convert - ops: typing.List[typing.Union[typing.Tuple[typing.Literal[0], None], typing.Tuple[int, int]]] = [(0, None)] - - internal_v: CastableType #typing.Union[CLVMStorage, typing.Tuple, typing.List] - target: int - element: CLVMStorage + stack = [v] + ops = [(0, None)] # convert while len(ops) > 0: - op_target = ops.pop() - - # this form allows mypy to follow the not-none-ness of op_target[1] for all other operations + op, target = ops.pop() # convert value - if op_target[1] is None: - assert op_target[0] == 0 + if op == 0: if looks_like_clvm_object(stack[-1]): continue - internal_v = stack.pop() - if isinstance(internal_v, tuple): - if len(internal_v) != 2: - raise ValueError("can't cast tuple of size %d" % len(internal_v)) - left, right = internal_v + v = stack.pop() + if isinstance(v, tuple): + if len(v) != 2: + raise ValueError("can't cast tuple of size %d" % len(v)) + left, right = v target = len(stack) - # if not looks_like_clvm_object(left) or not looks_like_clvm_object(right): - # assert False stack.append(CLVMObject((left, right))) if not looks_like_clvm_object(right): stack.append(right) @@ -102,47 +92,36 @@ def to_sexp_type( stack.append(left) ops.append((1, target)) # set left ops.append((0, None)) # convert - elif isinstance(internal_v, list): + continue + if isinstance(v, list): target = len(stack) stack.append(CLVMObject(NULL)) - for _ in internal_v: + for _ in v: stack.append(_) ops.append((3, target)) # prepend list # we only need to convert if it's not already the right # type if not looks_like_clvm_object(_): ops.append((0, None)) # convert - else: - # TODO: do we have to ignore? - stack.append(CLVMObject(convert_atom_to_bytes(internal_v))) # taype: ignore[arg-type] - elif op_target[0] == 1: # set left - target = op_target[1] - # TODO: do we have to ignore? - element = stack[target] # taype: ignore[assignment] - pair = element.pair - assert pair is not None - # TODO: do we have to ignore? - element.pair = (CLVMObject(stack.pop()), pair[1]) # taype: ignore[arg-type] - elif op_target[0] == 2: # set right - target = op_target[1] - # TODO: do we have to ignore? - element = stack[target] # taype: ignore[assignment] - pair = element.pair - assert pair is not None - # TODO: do we have to ignore? - element.pair = (pair[0], CLVMObject(stack.pop())) # taype: ignore[arg-type] - elif op_target[0] == 3: # prepend list - target = op_target[1] - # TODO: do we have to ignore? - stack[target] = CLVMObject((stack.pop(), stack[target])) # taype: ignore[arg-type] - # TODO: what about an else to fail explicitly on an unknown op? + continue + stack.append(CLVMObject(convert_atom_to_bytes(v))) + continue + + if op == 1: # set left + stack[target].pair = (CLVMObject(stack.pop()), stack[target].pair[1]) + continue + if op == 2: # set right + stack[target].pair = (stack[target].pair[0], CLVMObject(stack.pop())) + continue + if op == 3: # prepend list + stack[target] = CLVMObject((stack.pop(), stack[target])) + continue # there's exactly one item left at this point if len(stack) != 1: raise ValueError("internal error") # stack[0] implements the clvm object protocol and can be wrapped by an SExp - # TODO: do we have to ignore? - return stack[0] # taype: ignore[return-value] + return stack[0] _T_SExp = typing.TypeVar("_T_SExp", bound="SExp") From c3214f60738bb791af66082e6d3c0b2fc6a1009b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 15:00:21 -0500 Subject: [PATCH 36/56] tidy --- clvm/CLVMObject.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/clvm/CLVMObject.py b/clvm/CLVMObject.py index eb80aa46..04cffdf0 100644 --- a/clvm/CLVMObject.py +++ b/clvm/CLVMObject.py @@ -16,8 +16,6 @@ class CLVMStorage(typing.Protocol): _T_CLVMObject = typing.TypeVar("_T_CLVMObject", bound="CLVMObject") -if typing.TYPE_CHECKING: - from .SExp import CastableType class CLVMObject: """ @@ -29,7 +27,6 @@ class CLVMObject: # this is always a 2-tuple of an object implementing the CLVM object # protocol. - # pair: typing.Optional[typing.Tuple[CastableType, CastableType]] pair: typing.Optional[PairType] __slots__ = ["atom", "pair"] From 271ff395d24129876a2e33f2cee7ff3df43f6e49 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 16:44:02 -0500 Subject: [PATCH 37/56] tidy --- clvm/SExp.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index f1518b1d..157259a7 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -59,9 +59,6 @@ def convert_atom_to_bytes( ValType = typing.Union["SExp", CastableType] StackType = typing.List[ValType] -# StackValType = typing.Union[CLVMStorage, typing.Tuple[CLVMStorage, CLVMStorage]] -# StackType = typing.List[typing.Union[StackValType, "StackType"]] - # returns a clvm-object like object @typing.no_type_check From 6bd07201c457862acbdc6ed9e5dea3bc685015d4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 16:49:11 -0500 Subject: [PATCH 38/56] tidy --- tests/operatordict_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/operatordict_test.py b/tests/operatordict_test.py index a6315ab7..13594fd4 100644 --- a/tests/operatordict_test.py +++ b/tests/operatordict_test.py @@ -13,13 +13,10 @@ def test_operatordict_constructor(self) -> None: """ # ignoring because apparently it doesn't matter for this test that the types are all wrong d: Dict[bytes, OperatorProtocol] = {b"\01": "hello", b"\02": "goodbye"} # type: ignore [dict-item] - # TODO: or do we want to retain the AttributeError behavior? with self.assertRaises(AssertionError): OperatorDict(d) # type: ignore[call-overload] - # TODO: or do we want to retain the AttributeError behavior? with self.assertRaises(AssertionError): OperatorDict(d, apply=b"\01") # type: ignore[call-overload] - # TODO: or do we want to retain the AttributeError behavior? with self.assertRaises(AssertionError): OperatorDict(d, quote=b"\01") # type: ignore[call-overload] o = OperatorDict(d, apply=b"\01", quote=b"\02") From 0b30a6aef476fe46b730fd7736b679349449c8ec Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 17:04:33 -0500 Subject: [PATCH 39/56] prepare for more strict mypy --- mypy.ini | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 4e68bd74..56f3fa15 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,10 +1,20 @@ [mypy] files = clvm,tests,*.py show_error_codes = True +warn_unused_ignores = True +;disallow_any_generics = True +;disallow_subclassing_any = True +;disallow_untyped_calls = True disallow_untyped_defs = True +;disallow_incomplete_defs = True +;check_untyped_defs = True +;disallow_untyped_decorators = True no_implicit_optional = True -warn_unused_ignores = True +;warn_return_any = True +;no_implicit_reexport = True +;strict_equality = True +;warn_redundant_casts = True [mypy-blspy.*] ignore_missing_imports = True From 1a43f5f67285c32958161c68cf61d928af9cecc8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 17:05:41 -0500 Subject: [PATCH 40/56] enable 'free' mypy constraints --- mypy.ini | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mypy.ini b/mypy.ini index 56f3fa15..954d52f8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,17 +4,17 @@ show_error_codes = True warn_unused_ignores = True ;disallow_any_generics = True -;disallow_subclassing_any = True -;disallow_untyped_calls = True +disallow_subclassing_any = True +disallow_untyped_calls = True disallow_untyped_defs = True -;disallow_incomplete_defs = True -;check_untyped_defs = True -;disallow_untyped_decorators = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True no_implicit_optional = True -;warn_return_any = True +warn_return_any = True ;no_implicit_reexport = True -;strict_equality = True -;warn_redundant_casts = True +strict_equality = True +warn_redundant_casts = True [mypy-blspy.*] ignore_missing_imports = True From 1f9dccd290b9e1b09528ee6caa898d6e180fe69c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 17:07:06 -0500 Subject: [PATCH 41/56] no_implicit_reexport --- mypy.ini | 2 +- tests/as_python_test.py | 3 +-- tests/operators_test.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mypy.ini b/mypy.ini index 954d52f8..0d03e6da 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,7 +12,7 @@ check_untyped_defs = True disallow_untyped_decorators = True no_implicit_optional = True warn_return_any = True -;no_implicit_reexport = True +no_implicit_reexport = True strict_equality = True warn_redundant_casts = True diff --git a/tests/as_python_test.py b/tests/as_python_test.py index 8a202f9b..a55be354 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -2,9 +2,8 @@ from typing import List, Tuple, Union -from clvm import SExp from clvm.CLVMObject import CLVMObject -from clvm.SExp import CastableType +from clvm.SExp import CastableType, SExp from blspy import G1Element from clvm.EvalError import EvalError diff --git a/tests/operators_test.py b/tests/operators_test.py index dd2d87ef..5ac7c830 100644 --- a/tests/operators_test.py +++ b/tests/operators_test.py @@ -3,7 +3,7 @@ from clvm.operators import (OPERATOR_LOOKUP, KEYWORD_TO_ATOM, default_unknown_op, OperatorDict) from clvm.EvalError import EvalError -from clvm import SExp +from clvm.SExp import SExp from clvm.costs import CONCAT_BASE_COST From bdbb7377951da93360a15abf2f413f2a9a52c7a4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 19:58:53 -0500 Subject: [PATCH 42/56] some untyped generics --- clvm/SExp.py | 2 +- clvm/op_utils.py | 14 ++++++++------ tests/serialize_test.py | 5 ++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index 157259a7..4bc2a3cb 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -22,7 +22,7 @@ str, int, None, - list, + typing.Sequence["CastableType"], typing.Tuple["CastableType", "CastableType"], ] diff --git a/clvm/op_utils.py b/clvm/op_utils.py index 0bbca458..6ce3f7ba 100644 --- a/clvm/op_utils.py +++ b/clvm/op_utils.py @@ -1,12 +1,14 @@ import types -from typing import Callable, Dict, Optional +from typing import Dict, Optional + +from clvm.operators import OperatorProtocol def operators_for_dict( - keyword_to_atom: Dict, - op_dict: Dict[str, Callable], - op_name_lookup: Optional[Dict] = None, -) -> Dict: + keyword_to_atom: Dict[str, bytes], + op_dict: Dict[str, OperatorProtocol], + op_name_lookup: Optional[Dict[str, str]] = None, +) -> Dict[bytes, OperatorProtocol]: if op_name_lookup is None: op_name_lookup = {} @@ -19,7 +21,7 @@ def operators_for_dict( return d -def operators_for_module(keyword_to_atom: Dict, mod: types.ModuleType, op_name_lookup: Optional[Dict] = None) -> Dict: +def operators_for_module(keyword_to_atom: Dict[str, bytes], mod: types.ModuleType, op_name_lookup: Optional[Dict[str, str]] = None) -> Dict[bytes, OperatorProtocol]: if op_name_lookup is None: op_name_lookup = {} return operators_for_dict(keyword_to_atom, mod.__dict__, op_name_lookup) diff --git a/tests/serialize_test.py b/tests/serialize_test.py index 250b202b..b84deb1c 100644 --- a/tests/serialize_test.py +++ b/tests/serialize_test.py @@ -66,7 +66,10 @@ def test_short_lists(self) -> None: def test_cons_box(self) -> None: self.check_serde((None, None)) - self.check_serde((None, [1, 2, 30, 40, 600, (None, 18)])) + a: CastableType = (None, 18) + b: CastableType = [1, 2, 30, 40, 600, a] + c: CastableType = (None, b) + self.check_serde(c) self.check_serde((100, (TEXT, (30, (50, (90, (TEXT, TEXT + TEXT))))))) def test_long_blobs(self) -> None: From e66b640cb794fb67b7bddd73e2bde21883825045 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 14 Dec 2023 20:04:59 -0500 Subject: [PATCH 43/56] almost, but not quite --- clvm/op_utils.py | 6 +++++- clvm/serialize.py | 17 +++++++++-------- mypy.ini | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/clvm/op_utils.py b/clvm/op_utils.py index 6ce3f7ba..9ad66855 100644 --- a/clvm/op_utils.py +++ b/clvm/op_utils.py @@ -21,7 +21,11 @@ def operators_for_dict( return d -def operators_for_module(keyword_to_atom: Dict[str, bytes], mod: types.ModuleType, op_name_lookup: Optional[Dict[str, str]] = None) -> Dict[bytes, OperatorProtocol]: +def operators_for_module( + keyword_to_atom: Dict[str, bytes], + mod: types.ModuleType, + op_name_lookup: Optional[Dict[str, str]] = None, +) -> Dict[bytes, OperatorProtocol]: if op_name_lookup is None: op_name_lookup = {} return operators_for_dict(keyword_to_atom, mod.__dict__, op_name_lookup) diff --git a/clvm/serialize.py b/clvm/serialize.py index e60c09cd..55eee4fc 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -21,7 +21,7 @@ if typing.TYPE_CHECKING: - from .SExp import SExp + from .SExp import CastableType, SExp MAX_SINGLE_BYTE = 0x7F @@ -30,12 +30,13 @@ T = typing.TypeVar("T") +ToSExp = typing.Callable[["CastableType"], CLVMStorage] OpCallable = typing.Callable[ - ["OpStackType", "ValStackType", typing.BinaryIO, typing.Type], None + ["OpStackType", "ValStackType", typing.BinaryIO, ToSExp], None ] -ValStackType = typing.List["SExp"] +ValStackType = typing.List[CLVMStorage] OpStackType = typing.List[OpCallable] @@ -100,7 +101,7 @@ def sexp_to_stream(sexp: SExp, f: typing.BinaryIO) -> None: def _op_read_sexp( - op_stack: OpStackType, val_stack: ValStackType, f: typing.BinaryIO, to_sexp: typing.Callable[[bytes], SExp], + op_stack: OpStackType, val_stack: ValStackType, f: typing.BinaryIO, to_sexp: ToSExp, ) -> None: blob = f.read(1) if len(blob) == 0: @@ -118,14 +119,14 @@ def _op_cons( op_stack: OpStackType, val_stack: ValStackType, f: typing.BinaryIO, - to_sexp: typing.Callable[[typing.Tuple[SExp, SExp]], SExp], + to_sexp: ToSExp, ) -> None: right = val_stack.pop() left = val_stack.pop() val_stack.append(to_sexp((left, right))) -def sexp_from_stream(f: typing.BinaryIO, to_sexp: typing.Callable[[SExp], T]) -> T: +def sexp_from_stream(f: typing.BinaryIO, to_sexp: ToSExp) -> CLVMStorage: op_stack: OpStackType = [_op_read_sexp] val_stack: ValStackType = [] @@ -188,8 +189,8 @@ def sexp_buffer_from_stream(f: typing.BinaryIO) -> bytes: def _atom_from_stream( - f: typing.BinaryIO, b: int, to_sexp: typing.Callable[[bytes], T] -) -> T: + f: typing.BinaryIO, b: int, to_sexp: ToSExp +) -> CLVMStorage: if b == 0x80: return to_sexp(b"") if b <= MAX_SINGLE_BYTE: diff --git a/mypy.ini b/mypy.ini index 0d03e6da..a77ece31 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,7 +3,7 @@ files = clvm,tests,*.py show_error_codes = True warn_unused_ignores = True -;disallow_any_generics = True +disallow_any_generics = True disallow_subclassing_any = True disallow_untyped_calls = True disallow_untyped_defs = True From 80274437fce6600644630350c16010208d97f373 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 18 Dec 2023 15:24:55 -0800 Subject: [PATCH 44/56] ToCLVMStorage --- clvm/serialize.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/clvm/serialize.py b/clvm/serialize.py index 55eee4fc..9ee671e0 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -31,9 +31,12 @@ T = typing.TypeVar("T") ToSExp = typing.Callable[["CastableType"], CLVMStorage] +ToCLVMStorage = typing.Callable[ + [typing.Union[bytes, typing.Tuple[CLVMStorage, CLVMStorage]]], CLVMStorage +] OpCallable = typing.Callable[ - ["OpStackType", "ValStackType", typing.BinaryIO, ToSExp], None + ["OpStackType", "ValStackType", typing.BinaryIO, ToCLVMStorage], None ] ValStackType = typing.List[CLVMStorage] @@ -101,7 +104,10 @@ def sexp_to_stream(sexp: SExp, f: typing.BinaryIO) -> None: def _op_read_sexp( - op_stack: OpStackType, val_stack: ValStackType, f: typing.BinaryIO, to_sexp: ToSExp, + op_stack: OpStackType, + val_stack: ValStackType, + f: typing.BinaryIO, + to_sexp: ToCLVMStorage, ) -> None: blob = f.read(1) if len(blob) == 0: @@ -119,7 +125,7 @@ def _op_cons( op_stack: OpStackType, val_stack: ValStackType, f: typing.BinaryIO, - to_sexp: ToSExp, + to_sexp: ToCLVMStorage, ) -> None: right = val_stack.pop() left = val_stack.pop() @@ -189,7 +195,7 @@ def sexp_buffer_from_stream(f: typing.BinaryIO) -> bytes: def _atom_from_stream( - f: typing.BinaryIO, b: int, to_sexp: ToSExp + f: typing.BinaryIO, b: int, to_sexp: ToCLVMStorage ) -> CLVMStorage: if b == 0x80: return to_sexp(b"") From 98e1f9cffc3a8638362e039cbba00411140e929d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 18 Dec 2023 19:05:36 -0500 Subject: [PATCH 45/56] break circular import --- clvm/op_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/clvm/op_utils.py b/clvm/op_utils.py index 9ad66855..fd5f126f 100644 --- a/clvm/op_utils.py +++ b/clvm/op_utils.py @@ -1,7 +1,10 @@ +from __future__ import annotations + import types -from typing import Dict, Optional +from typing import Dict, Optional, TYPE_CHECKING -from clvm.operators import OperatorProtocol +if TYPE_CHECKING: + from clvm.operators import OperatorProtocol def operators_for_dict( From 8369c45deea481eb396a5bd988e184057693a37b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 21 Dec 2023 15:31:16 -0500 Subject: [PATCH 46/56] touchup --- clvm/SExp.py | 3 ++- clvm/serialize.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index 4bc2a3cb..27051732 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -18,6 +18,7 @@ CastableType = typing.Union[ "SExp", CLVMStorage, + typing.SupportsBytes, bytes, str, int, @@ -211,7 +212,7 @@ def rest(self: _T_SExp) -> _T_SExp: def null(class_) -> "SExp": return class_.__null__ - def as_iter(self: _T_SExp) -> typing.Iterable[_T_SExp]: + def as_iter(self: _T_SExp) -> typing.Iterator[_T_SExp]: v = self while not v.nullp(): yield v.first() diff --git a/clvm/serialize.py b/clvm/serialize.py index 9ee671e0..ad7c3016 100644 --- a/clvm/serialize.py +++ b/clvm/serialize.py @@ -30,7 +30,6 @@ T = typing.TypeVar("T") -ToSExp = typing.Callable[["CastableType"], CLVMStorage] ToCLVMStorage = typing.Callable[ [typing.Union[bytes, typing.Tuple[CLVMStorage, CLVMStorage]]], CLVMStorage ] @@ -132,7 +131,7 @@ def _op_cons( val_stack.append(to_sexp((left, right))) -def sexp_from_stream(f: typing.BinaryIO, to_sexp: ToSExp) -> CLVMStorage: +def sexp_from_stream(f: typing.BinaryIO, to_sexp: typing.Callable[["CastableType"], T]) -> T: op_stack: OpStackType = [_op_read_sexp] val_stack: ValStackType = [] From 7222f093be62676bf1ce1de2059985b7bc11fdf5 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 19 Mar 2024 16:24:26 -0400 Subject: [PATCH 47/56] future --- clvm/SExp.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index 27051732..a5e4b818 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import io import typing @@ -16,15 +18,15 @@ CastableType = typing.Union[ - "SExp", + SExp, CLVMStorage, typing.SupportsBytes, bytes, str, int, None, - typing.Sequence["CastableType"], - typing.Tuple["CastableType", "CastableType"], + typing.Sequence[CastableType], + typing.Tuple[CastableType, CastableType], ] @@ -57,7 +59,7 @@ def convert_atom_to_bytes( raise ValueError("can't cast %s (%s) to bytes" % (type(v), v)) -ValType = typing.Union["SExp", CastableType] +ValType = typing.Union[SExp, CastableType] StackType = typing.List[ValType] @@ -139,9 +141,9 @@ class SExp: elements implementing the CLVM object protocol. Exactly one of "atom" and "pair" must be None. """ - true: typing.ClassVar["SExp"] - false: typing.ClassVar["SExp"] - __null__: typing.ClassVar["SExp"] + true: typing.ClassVar[SExp] + false: typing.ClassVar[SExp] + __null__: typing.ClassVar[SExp] # the underlying object implementing the clvm object protocol atom: typing.Optional[bytes] @@ -155,7 +157,7 @@ def __init__(self, obj: CLVMStorage) -> None: self.pair = obj.pair # this returns a tuple of two SExp objects, or None - def as_pair(self) -> typing.Optional[typing.Tuple["SExp", "SExp"]]: + def as_pair(self) -> typing.Optional[typing.Tuple[SExp, SExp]]: pair = self.pair if pair is None: return pair @@ -209,7 +211,7 @@ def rest(self: _T_SExp) -> _T_SExp: raise EvalError("rest of non-cons", self) @classmethod - def null(class_) -> "SExp": + def null(class_) -> SExp: return class_.__null__ def as_iter(self: _T_SExp) -> typing.Iterator[_T_SExp]: From 23e47d85183139b8b6e17f2831040a40bcc7c5cd Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 19 Mar 2024 16:28:06 -0400 Subject: [PATCH 48/56] less future --- clvm/SExp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index a5e4b818..a561c6d6 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -18,15 +18,15 @@ CastableType = typing.Union[ - SExp, + "SExp", CLVMStorage, typing.SupportsBytes, bytes, str, int, None, - typing.Sequence[CastableType], - typing.Tuple[CastableType, CastableType], + typing.Sequence["CastableType"], + typing.Tuple["CastableType", "CastableType"], ] @@ -59,7 +59,7 @@ def convert_atom_to_bytes( raise ValueError("can't cast %s (%s) to bytes" % (type(v), v)) -ValType = typing.Union[SExp, CastableType] +ValType = typing.Union["SExp", CastableType] StackType = typing.List[ValType] From 8f05894814b769e6191f7a4b5c27672b25199665 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 21 Mar 2024 15:13:21 -0400 Subject: [PATCH 49/56] `SExp.to(Any)` --- clvm/SExp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index a561c6d6..0f22a99f 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -184,8 +184,9 @@ def as_bin(self) -> bytes: sexp_to_stream(self, f) return f.getvalue() + # TODO: should be `v: CastableType` @classmethod - def to(cls: typing.Type[_T_SExp], v: CastableType) -> _T_SExp: + def to(cls: typing.Type[_T_SExp], v: typing.Any) -> _T_SExp: if isinstance(v, cls): return v From 15f4c2e303a63307c5059b7e3bc087fcb3e846ed Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Thu, 21 Mar 2024 21:47:29 -0400 Subject: [PATCH 50/56] remove ignores --- tests/as_python_test.py | 6 +++--- tests/to_sexp_test.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/as_python_test.py b/tests/as_python_test.py index 2303055c..5c5c0d3d 100644 --- a/tests/as_python_test.py +++ b/tests/as_python_test.py @@ -187,14 +187,14 @@ def test_long_list(self) -> None: def test_invalid_type(self) -> None: with self.assertRaises(ValueError): - SExp.to(dummy_class) # type: ignore[arg-type] + SExp.to(dummy_class) def test_invalid_tuple(self) -> None: with self.assertRaises(ValueError): - SExp.to((dummy_class, dummy_class)) # type: ignore[arg-type] + SExp.to((dummy_class, dummy_class)) with self.assertRaises(ValueError): - SExp.to((dummy_class, dummy_class, dummy_class)) # type: ignore[arg-type] + SExp.to((dummy_class, dummy_class, dummy_class)) def test_clvm_object_tuple(self) -> None: o1 = CLVMObject(b"foo") diff --git a/tests/to_sexp_test.py b/tests/to_sexp_test.py index 06b6dbca..2d1e646d 100644 --- a/tests/to_sexp_test.py +++ b/tests/to_sexp_test.py @@ -179,7 +179,7 @@ def test_empty_list_conversions(self) -> None: def test_eager_conversion(self) -> None: with self.assertRaises(ValueError): - SExp.to(("foobar", (1, {}))) # type: ignore[arg-type] + SExp.to(("foobar", (1, {}))) def test_convert_atom(self) -> None: assert convert_atom_to_bytes(0x133742) == bytes([0x13, 0x37, 0x42]) From f806e686a531b98e695b00b6177d34d589fcbb2d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 22 Mar 2024 09:54:17 -0400 Subject: [PATCH 51/56] use `PythonReturnType` for as python return type --- clvm/SExp.py | 6 +++--- clvm/as_python.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index 0f22a99f..deca3366 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -5,7 +5,7 @@ import typing_extensions -from .as_python import as_python +from .as_python import PythonReturnType, as_python from .CLVMObject import CLVMObject, CLVMStorage from .EvalError import EvalError @@ -196,7 +196,7 @@ def to(cls: typing.Type[_T_SExp], v: typing.Any) -> _T_SExp: # this will lazily convert elements return cls(to_sexp_type(v)) - def cons(self: _T_SExp, right: _T_SExp) -> _T_SExp: + def cons(self: _T_SExp, right: CastableType) -> _T_SExp: return self.to((self, right)) def first(self: _T_SExp) -> _T_SExp: @@ -249,7 +249,7 @@ def list_len(self) -> int: v = v.rest() return size - def as_python(self) -> typing.Any: + def as_python(self) -> PythonReturnType: return as_python(self) def __str__(self) -> str: diff --git a/clvm/as_python.py b/clvm/as_python.py index 41ecf9a9..c7f9ee98 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -53,10 +53,12 @@ def _as_python(op_stack: OpStackType, val_stack: ValStackType) -> None: val_stack.append(t.atom) # type:ignore[arg-type] -def as_python(sexp: SExp) -> Any: +def as_python(sexp: SExp) -> PythonReturnType: op_stack: OpStackType = [_as_python] val_stack: ValStackType = [sexp] while op_stack: op_f = op_stack.pop() op_f(op_stack, val_stack) - return val_stack[-1] + result = val_stack[-1] + assert not isinstance(result, SExp) + return result From 1f3f89bc5013f1b0f45203882a2f4fad8a82dea4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 22 Mar 2024 10:00:20 -0400 Subject: [PATCH 52/56] drop any --- clvm/as_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clvm/as_python.py b/clvm/as_python.py index c7f9ee98..11656aa3 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Callable, List, Tuple, TYPE_CHECKING, Union +from typing import Callable, List, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: from clvm.SExp import SExp From bfe9e84ceb61ee0e7cd095c59141aef9f239a199 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Fri, 22 Mar 2024 11:07:44 -0400 Subject: [PATCH 53/56] hmm --- clvm/as_python.py | 1 + 1 file changed, 1 insertion(+) diff --git a/clvm/as_python.py b/clvm/as_python.py index 11656aa3..02e64e73 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -60,5 +60,6 @@ def as_python(sexp: SExp) -> PythonReturnType: op_f = op_stack.pop() op_f(op_stack, val_stack) result = val_stack[-1] + from clvm.SExp import SExp assert not isinstance(result, SExp) return result From 44ca118975de2a77102d6a7b9c47f4333b495f54 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 25 Mar 2024 14:06:01 -0400 Subject: [PATCH 54/56] Revert "hmm" This reverts commit bfe9e84ceb61ee0e7cd095c59141aef9f239a199. --- clvm/as_python.py | 1 - 1 file changed, 1 deletion(-) diff --git a/clvm/as_python.py b/clvm/as_python.py index 02e64e73..11656aa3 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -60,6 +60,5 @@ def as_python(sexp: SExp) -> PythonReturnType: op_f = op_stack.pop() op_f(op_stack, val_stack) result = val_stack[-1] - from clvm.SExp import SExp assert not isinstance(result, SExp) return result From 8f7c410fe8d294654b1576d6975ef2198a18557b Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 25 Mar 2024 14:06:31 -0400 Subject: [PATCH 55/56] Revert "drop any" This reverts commit 1f3f89bc5013f1b0f45203882a2f4fad8a82dea4. --- clvm/as_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clvm/as_python.py b/clvm/as_python.py index 11656aa3..c7f9ee98 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Callable, List, Tuple, TYPE_CHECKING, Union +from typing import Any, Callable, List, Tuple, TYPE_CHECKING, Union if TYPE_CHECKING: from clvm.SExp import SExp From 7a956fc0fad0cdca67cfc4062c48ef2854a528b8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 25 Mar 2024 14:06:49 -0400 Subject: [PATCH 56/56] Revert "use `PythonReturnType` for as python return type" This reverts commit f806e686a531b98e695b00b6177d34d589fcbb2d. --- clvm/SExp.py | 6 +++--- clvm/as_python.py | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/clvm/SExp.py b/clvm/SExp.py index deca3366..0f22a99f 100644 --- a/clvm/SExp.py +++ b/clvm/SExp.py @@ -5,7 +5,7 @@ import typing_extensions -from .as_python import PythonReturnType, as_python +from .as_python import as_python from .CLVMObject import CLVMObject, CLVMStorage from .EvalError import EvalError @@ -196,7 +196,7 @@ def to(cls: typing.Type[_T_SExp], v: typing.Any) -> _T_SExp: # this will lazily convert elements return cls(to_sexp_type(v)) - def cons(self: _T_SExp, right: CastableType) -> _T_SExp: + def cons(self: _T_SExp, right: _T_SExp) -> _T_SExp: return self.to((self, right)) def first(self: _T_SExp) -> _T_SExp: @@ -249,7 +249,7 @@ def list_len(self) -> int: v = v.rest() return size - def as_python(self) -> PythonReturnType: + def as_python(self) -> typing.Any: return as_python(self) def __str__(self) -> str: diff --git a/clvm/as_python.py b/clvm/as_python.py index c7f9ee98..41ecf9a9 100644 --- a/clvm/as_python.py +++ b/clvm/as_python.py @@ -53,12 +53,10 @@ def _as_python(op_stack: OpStackType, val_stack: ValStackType) -> None: val_stack.append(t.atom) # type:ignore[arg-type] -def as_python(sexp: SExp) -> PythonReturnType: +def as_python(sexp: SExp) -> Any: op_stack: OpStackType = [_as_python] val_stack: ValStackType = [sexp] while op_stack: op_f = op_stack.pop() op_f(op_stack, val_stack) - result = val_stack[-1] - assert not isinstance(result, SExp) - return result + return val_stack[-1]