From 4f47497450044fdb9692881862477466348bad67 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Fri, 4 Oct 2024 20:35:20 +0300 Subject: [PATCH 01/13] fix[venom]: remove duplicate volatile instructions (#4263) remove duplicate `assert`, `assert_unreachable` from `VOLATILE_INSTRUCTIONS` dictionary --- vyper/venom/basicblock.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 1199579b3f..45db8b232f 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -21,8 +21,6 @@ "istore", "tload", "tstore", - "assert", - "assert_unreachable", "mstore", "mload", "calldatacopy", From 96551197701251ebde92f243fc900eed2005cee3 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 4 Oct 2024 15:43:52 -0400 Subject: [PATCH 02/13] feat[tool]: add integrity hash to initcode (#4234) this commit adds the integrity hash of the source code to the initcode. it extends the existing cbor metadata payload in the initcode, so that verifiers can compare the integrity hash to the artifact produced by a source bundle. the integrity hash is put in the initcode to preserve bytecode space of the runtime code. refactor: - change existing `insert_compiler_metadata=` flag to the more generic `compiler_metadata=None`, which is more extensible. --- .../builtins/codegen/test_raw_call.py | 18 ++++++++- tests/unit/compiler/test_bytecode_runtime.py | 37 ++++++++++++++----- vyper/compiler/output.py | 6 +-- vyper/compiler/phases.py | 17 +++++---- vyper/ir/compile_ir.py | 15 +++++--- 5 files changed, 64 insertions(+), 29 deletions(-) diff --git a/tests/functional/builtins/codegen/test_raw_call.py b/tests/functional/builtins/codegen/test_raw_call.py index 4107f9a4d0..bf953ff018 100644 --- a/tests/functional/builtins/codegen/test_raw_call.py +++ b/tests/functional/builtins/codegen/test_raw_call.py @@ -261,6 +261,12 @@ def __default__(): assert env.message_call(caller.address, data=sig) == b"" +def _strip_initcode_suffix(bytecode): + bs = bytes.fromhex(bytecode.removeprefix("0x")) + to_strip = int.from_bytes(bs[-2:], "big") + return bs[:-to_strip].hex() + + # check max_outsize=0 does same thing as not setting max_outsize. # compile to bytecode and compare bytecode directly. def test_max_outsize_0(): @@ -276,7 +282,11 @@ def test_raw_call(_target: address): """ output1 = compile_code(code1, output_formats=["bytecode", "bytecode_runtime"]) output2 = compile_code(code2, output_formats=["bytecode", "bytecode_runtime"]) - assert output1 == output2 + assert output1["bytecode_runtime"] == output2["bytecode_runtime"] + + bytecode1 = output1["bytecode"] + bytecode2 = output2["bytecode"] + assert _strip_initcode_suffix(bytecode1) == _strip_initcode_suffix(bytecode2) # check max_outsize=0 does same thing as not setting max_outsize, @@ -298,7 +308,11 @@ def test_raw_call(_target: address) -> bool: """ output1 = compile_code(code1, output_formats=["bytecode", "bytecode_runtime"]) output2 = compile_code(code2, output_formats=["bytecode", "bytecode_runtime"]) - assert output1 == output2 + assert output1["bytecode_runtime"] == output2["bytecode_runtime"] + + bytecode1 = output1["bytecode"] + bytecode2 = output2["bytecode"] + assert _strip_initcode_suffix(bytecode1) == _strip_initcode_suffix(bytecode2) # test functionality of max_outsize=0 diff --git a/tests/unit/compiler/test_bytecode_runtime.py b/tests/unit/compiler/test_bytecode_runtime.py index 213adce017..1d38130c49 100644 --- a/tests/unit/compiler/test_bytecode_runtime.py +++ b/tests/unit/compiler/test_bytecode_runtime.py @@ -55,13 +55,17 @@ def test_bytecode_runtime(): def test_bytecode_signature(): - out = vyper.compile_code(simple_contract_code, output_formats=["bytecode_runtime", "bytecode"]) + out = vyper.compile_code( + simple_contract_code, output_formats=["bytecode_runtime", "bytecode", "integrity"] + ) runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x")) initcode = bytes.fromhex(out["bytecode"].removeprefix("0x")) metadata = _parse_cbor_metadata(initcode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) assert data_section_lengths == [] @@ -73,14 +77,18 @@ def test_bytecode_signature_dense_jumptable(): settings = Settings(optimize=OptimizationLevel.CODESIZE) out = vyper.compile_code( - many_functions, output_formats=["bytecode_runtime", "bytecode"], settings=settings + many_functions, + output_formats=["bytecode_runtime", "bytecode", "integrity"], + settings=settings, ) runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x")) initcode = bytes.fromhex(out["bytecode"].removeprefix("0x")) metadata = _parse_cbor_metadata(initcode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) assert data_section_lengths == [5, 35] @@ -92,14 +100,18 @@ def test_bytecode_signature_sparse_jumptable(): settings = Settings(optimize=OptimizationLevel.GAS) out = vyper.compile_code( - many_functions, output_formats=["bytecode_runtime", "bytecode"], settings=settings + many_functions, + output_formats=["bytecode_runtime", "bytecode", "integrity"], + settings=settings, ) runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x")) initcode = bytes.fromhex(out["bytecode"].removeprefix("0x")) metadata = _parse_cbor_metadata(initcode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) assert data_section_lengths == [8] @@ -108,13 +120,17 @@ def test_bytecode_signature_sparse_jumptable(): def test_bytecode_signature_immutables(): - out = vyper.compile_code(has_immutables, output_formats=["bytecode_runtime", "bytecode"]) + out = vyper.compile_code( + has_immutables, output_formats=["bytecode_runtime", "bytecode", "integrity"] + ) runtime_code = bytes.fromhex(out["bytecode_runtime"].removeprefix("0x")) initcode = bytes.fromhex(out["bytecode"].removeprefix("0x")) metadata = _parse_cbor_metadata(initcode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + assert integrity_hash.hex() == out["integrity"] assert runtime_len == len(runtime_code) assert data_section_lengths == [] @@ -129,7 +145,10 @@ def test_bytecode_signature_deployed(code, get_contract, env): deployed_code = env.get_code(c.address) metadata = _parse_cbor_metadata(c.bytecode) - runtime_len, data_section_lengths, immutables_len, compiler = metadata + integrity_hash, runtime_len, data_section_lengths, immutables_len, compiler = metadata + + out = vyper.compile_code(code, output_formats=["integrity"]) + assert integrity_hash.hex() == out["integrity"] assert compiler == {"vyper": list(vyper.version.version_tuple)} diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 577afd3822..09d299b90d 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -320,15 +320,13 @@ def _build_source_map_output(compiler_data, bytecode, pc_maps): def build_source_map_output(compiler_data: CompilerData) -> dict: - bytecode, pc_maps = compile_ir.assembly_to_evm( - compiler_data.assembly, insert_compiler_metadata=False - ) + bytecode, pc_maps = compile_ir.assembly_to_evm(compiler_data.assembly, compiler_metadata=None) return _build_source_map_output(compiler_data, bytecode, pc_maps) def build_source_map_runtime_output(compiler_data: CompilerData) -> dict: bytecode, pc_maps = compile_ir.assembly_to_evm( - compiler_data.assembly_runtime, insert_compiler_metadata=False + compiler_data.assembly_runtime, compiler_metadata=None ) return _build_source_map_output(compiler_data, bytecode, pc_maps) diff --git a/vyper/compiler/phases.py b/vyper/compiler/phases.py index 147af24d67..97df73cdae 100644 --- a/vyper/compiler/phases.py +++ b/vyper/compiler/phases.py @@ -2,7 +2,7 @@ import warnings from functools import cached_property from pathlib import Path, PurePath -from typing import Optional +from typing import Any, Optional from vyper import ast as vy_ast from vyper.ast import natspec @@ -249,12 +249,15 @@ def assembly_runtime(self) -> list: @cached_property def bytecode(self) -> bytes: - insert_compiler_metadata = not self.no_bytecode_metadata - return generate_bytecode(self.assembly, insert_compiler_metadata=insert_compiler_metadata) + metadata = None + if not self.no_bytecode_metadata: + module_t = self.compilation_target._metadata["type"] + metadata = bytes.fromhex(module_t.integrity_sum) + return generate_bytecode(self.assembly, compiler_metadata=metadata) @cached_property def bytecode_runtime(self) -> bytes: - return generate_bytecode(self.assembly_runtime, insert_compiler_metadata=False) + return generate_bytecode(self.assembly_runtime, compiler_metadata=None) @cached_property def blueprint_bytecode(self) -> bytes: @@ -351,7 +354,7 @@ def _find_nested_opcode(assembly, key): return any(_find_nested_opcode(x, key) for x in sublists) -def generate_bytecode(assembly: list, insert_compiler_metadata: bool) -> bytes: +def generate_bytecode(assembly: list, compiler_metadata: Optional[Any]) -> bytes: """ Generate bytecode from assembly instructions. @@ -365,6 +368,4 @@ def generate_bytecode(assembly: list, insert_compiler_metadata: bool) -> bytes: bytes Final compiled bytecode. """ - return compile_ir.assembly_to_evm(assembly, insert_compiler_metadata=insert_compiler_metadata)[ - 0 - ] + return compile_ir.assembly_to_evm(assembly, compiler_metadata=compiler_metadata)[0] diff --git a/vyper/ir/compile_ir.py b/vyper/ir/compile_ir.py index 4c68aa2c8f..2cc951b188 100644 --- a/vyper/ir/compile_ir.py +++ b/vyper/ir/compile_ir.py @@ -1155,22 +1155,24 @@ def _relocate_segments(assembly): # TODO: change API to split assembly_to_evm and assembly_to_source/symbol_maps -def assembly_to_evm(assembly, pc_ofst=0, insert_compiler_metadata=False): +def assembly_to_evm(assembly, pc_ofst=0, compiler_metadata=None): bytecode, source_maps, _ = assembly_to_evm_with_symbol_map( - assembly, pc_ofst=pc_ofst, insert_compiler_metadata=insert_compiler_metadata + assembly, pc_ofst=pc_ofst, compiler_metadata=compiler_metadata ) return bytecode, source_maps -def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, insert_compiler_metadata=False): +def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, compiler_metadata=None): """ Assembles assembly into EVM assembly: list of asm instructions pc_ofst: when constructing the source map, the amount to offset all pcs by (no effect until we add deploy code source map) - insert_compiler_metadata: whether to append vyper metadata to output - (should be true for runtime code) + compiler_metadata: any compiler metadata to add. pass `None` to indicate + no metadata to be added (should always be `None` for + runtime code). the value is opaque, and will be passed + directly to `cbor2.dumps()`. """ line_number_map = { "breakpoints": set(), @@ -1278,10 +1280,11 @@ def assembly_to_evm_with_symbol_map(assembly, pc_ofst=0, insert_compiler_metadat pc += 1 bytecode_suffix = b"" - if insert_compiler_metadata: + if compiler_metadata is not None: # this will hold true when we are in initcode assert immutables_len is not None metadata = ( + compiler_metadata, len(runtime_code), data_section_lengths, immutables_len, From 5d8280feec16f86ae1a888e770f20a96113fdabd Mon Sep 17 00:00:00 2001 From: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:50:02 +0000 Subject: [PATCH 03/13] fix[venom]: fix `_stack_reorder()` routine (#4220) fix an issue where `stack_reorder()` reorders operands incorrectly, resulting in the result stack not matching the target stack. this bug can manifest when there are multiple copies of an operand on the stack. in the `stack_reorder()` loop, an operand gets moved past one of its copies which has not been moved yet, resulting in the operand getting moved twice, instead of each copy of the operand getting moved once, since `get_depth()` returns the wrong copy of the operand after the first move. this commit fixes the issue by keeping track of the positions of each copy of each stack item, and ensuring that each copy only gets moved once. --------- Co-authored-by: Charles Cooper --- .../unit/compiler/venom/test_stack_reorder.py | 28 +++++++++++++ vyper/venom/venom_to_assembly.py | 39 +++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 tests/unit/compiler/venom/test_stack_reorder.py diff --git a/tests/unit/compiler/venom/test_stack_reorder.py b/tests/unit/compiler/venom/test_stack_reorder.py new file mode 100644 index 0000000000..a9f505984e --- /dev/null +++ b/tests/unit/compiler/venom/test_stack_reorder.py @@ -0,0 +1,28 @@ +from vyper.venom import generate_assembly_experimental +from vyper.venom.context import IRContext + + +def test_stack_reorder(): + """ + Test to was created from the example in the + issue https://github.com/vyperlang/vyper/issues/4215 + this example should fail with original stack reorder + algorithm but succeed with new one + """ + ctx = IRContext() + fn = ctx.create_function("_global") + + bb = fn.get_basic_block() + var0 = bb.append_instruction("store", 1) + var1 = bb.append_instruction("store", 2) + var2 = bb.append_instruction("store", 3) + var3 = bb.append_instruction("store", 4) + var4 = bb.append_instruction("store", 5) + + bb.append_instruction("staticcall", var0, var1, var2, var3, var4, var3) + + ret_val = bb.append_instruction("add", var4, var4) + + bb.append_instruction("ret", ret_val) + + generate_assembly_experimental(ctx) diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 390fab8e7c..9de75dab38 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -1,4 +1,5 @@ -from collections import Counter +from bisect import insort +from collections import Counter, defaultdict from typing import Any from vyper.exceptions import CompilerPanic, StackTooDeep @@ -205,14 +206,29 @@ def _stack_reorder( stack = stack.copy() stack_ops_count = len(stack_ops) + if stack_ops_count == 0: + return 0 counts = Counter(stack_ops) + # positions stores the positions of relevant operands + # on stack for example operand %82 is on positions [0, 3] + # this operand could ocure even more deeper in the stack + # but only those that are needed/relevant in calculation + # are considered + positions: dict[IROperand, list[int]] = defaultdict(list) + for op in stack_ops: + positions[op] = [] + for i in range(counts[op]): + positions[op].append(stack.get_depth(op, i + 1)) + for i in range(stack_ops_count): op = stack_ops[i] final_stack_depth = -(stack_ops_count - i - 1) - depth = stack.get_depth(op, counts[op]) # type: ignore - counts[op] -= 1 + depth = positions[op].pop() # type: ignore + assert depth not in range( + -stack_ops_count + 1, final_stack_depth + ), f"{depth} : ({-stack_ops_count - 1}, {final_stack_depth})" if depth == StackModel.NOT_IN_STACK: raise CompilerPanic(f"Variable {op} not in stack") @@ -223,9 +239,26 @@ def _stack_reorder( if op == stack.peek(final_stack_depth): continue + # moves the top item to original position + top_item_positions = positions[stack.peek(0)] + if len(top_item_positions) != 0: + top_item_positions.remove(0) + insort(top_item_positions, depth) + cost += self.swap(assembly, stack, depth) + + # moves the item from final position to top + final_item_positions = positions[stack.peek(final_stack_depth)] + if final_stack_depth in final_item_positions: + final_item_positions.remove(final_stack_depth) + final_item_positions.insert(0, 0) + else: + final_item_positions.insert(0, 0) + cost += self.swap(assembly, stack, final_stack_depth) + assert stack._stack[-len(stack_ops) :] == stack_ops, (stack, stack_ops) + return cost def _emit_input_operands( From 0f809c6f61c36b02b5f9dbae67be01cdc4c6e0f1 Mon Sep 17 00:00:00 2001 From: Daniel Schiavini Date: Fri, 4 Oct 2024 22:14:03 +0200 Subject: [PATCH 04/13] chore[docs]: add binary installation methods (#4258) - Add instructions to install the binaries from github releases to the documentation - Show pip before docker, as it might be more common and easier to install - Update python requirement in the installation docs --------- Co-authored-by: Charles Cooper --- docs/installing-vyper.rst | 69 ++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/docs/installing-vyper.rst b/docs/installing-vyper.rst index 8eaa93590a..515d88f87c 100644 --- a/docs/installing-vyper.rst +++ b/docs/installing-vyper.rst @@ -7,37 +7,16 @@ any errors. .. note:: - The easiest way to experiment with the language is to use the `Remix online compiler `_. - (Activate the vyper-remix plugin in the Plugin manager.) + The easiest way to experiment with the language is to use either `Try Vyper! `_ (maintained by the Vyper team) or the `Remix online compiler `_ (maintained by the Ethereum Foundation). + - To use Try Vyper, go to https://try.vyperlang.org and log in (requires Github login). + - To use remix, go to https://remix.ethereum.org and activate the vyper-remix plugin in the Plugin manager. -Docker -****** - -Vyper can be downloaded as docker image from `dockerhub `_: -:: - - docker pull vyperlang/vyper - -To run the compiler use the ``docker run`` command: -:: - - docker run -v $(pwd):/code vyperlang/vyper /code/ - -Alternatively you can log into the docker image and execute vyper on the prompt. -:: - - docker run -v $(pwd):/code/ -it --entrypoint /bin/bash vyperlang/vyper - root@d35252d1fb1b:/code# vyper - -The normal parameters are also supported, for example: -:: +Binaries +******** - docker run -v $(pwd):/code vyperlang/vyper -f abi /code/ - [{'name': 'test1', 'outputs': [], 'inputs': [{'type': 'uint256', 'name': 'a'}, {'type': 'bytes', 'name': 'b'}], 'constant': False, 'payable': False, 'type': 'function', 'gas': 441}, {'name': 'test2', 'outputs': [], 'inputs': [{'type': 'uint256', 'name': 'a'}], 'constant': False, 'payable': False, 'type': 'function', 'gas': 316}] +Vyper binaries for Windows, Mac and Linux are available for download from the `GitHub releases page +`_. -.. note:: - - If you would like to know how to install Docker, please follow their `documentation `_. PIP *** @@ -45,7 +24,7 @@ PIP Installing Python ================= -Vyper can only be built using Python 3.6 and higher. If you need to know how to install the correct version of python, +Vyper can only be built using Python 3.10 and higher. If you need to know how to install the correct version of python, follow the instructions from the official `Python website `_. Creating a virtual environment @@ -76,13 +55,43 @@ Each tagged version of vyper is uploaded to `pypi `_: +:: + + docker pull vyperlang/vyper + +To run the compiler use the ``docker run`` command: +:: + + docker run -v $(pwd):/code vyperlang/vyper /code/ + +Alternatively you can log into the docker image and execute vyper on the prompt. +:: + + docker run -v $(pwd):/code/ -it --entrypoint /bin/bash vyperlang/vyper + root@d35252d1fb1b:/code# vyper + +The normal parameters are also supported, for example: +:: + + docker run -v $(pwd):/code vyperlang/vyper -f abi /code/ + [{'name': 'test1', 'outputs': [], 'inputs': [{'type': 'uint256', 'name': 'a'}, {'type': 'bytes', 'name': 'b'}], 'constant': False, 'payable': False, 'type': 'function', 'gas': 441}, {'name': 'test2', 'outputs': [], 'inputs': [{'type': 'uint256', 'name': 'a'}], 'constant': False, 'payable': False, 'type': 'function', 'gas': 316}] + +.. note:: + + If you would like to know how to install Docker, please follow their `documentation `_. + nix *** From c7669bd2ebe2c405aa5572b58311b51517568143 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 4 Oct 2024 16:16:33 -0400 Subject: [PATCH 05/13] fix[ux]: fix relpath compiler panic on windows (#4228) fix a bug where `os.path.relpath()` raises an exception on window - when the source path and the destination path are on different drives. this commit introduces the helper function `safe_relpath()`, which tries hard to construct a relpath (using `os.path.relpath()`), but falls back to the original path (which might be an absolute path) instead of raising an exception. references: - https://docs.python.org/3/library/os.path.html#os.path.relpath --- vyper/compiler/output_bundle.py | 9 ++++----- vyper/semantics/analysis/module.py | 5 ++--- vyper/utils.py | 10 ++++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/vyper/compiler/output_bundle.py b/vyper/compiler/output_bundle.py index 92494e3a70..06a84064a1 100644 --- a/vyper/compiler/output_bundle.py +++ b/vyper/compiler/output_bundle.py @@ -1,7 +1,6 @@ import importlib import io import json -import os import zipfile from dataclasses import dataclass from functools import cached_property @@ -13,7 +12,7 @@ from vyper.compiler.settings import Settings from vyper.exceptions import CompilerPanic from vyper.semantics.analysis.module import _is_builtin -from vyper.utils import get_long_version +from vyper.utils import get_long_version, safe_relpath # data structures and routines for constructing "output bundles", # basically reproducible builds of a vyper contract, with varying @@ -62,7 +61,7 @@ def compiler_inputs(self) -> dict[str, CompilerInput]: sources = {} for c in inputs: - path = os.path.relpath(c.resolved_path) + path = safe_relpath(c.resolved_path) # note: there should be a 1:1 correspondence between # resolved_path and source_id, but for clarity use resolved_path # since it corresponds more directly to search path semantics. @@ -73,7 +72,7 @@ def compiler_inputs(self) -> dict[str, CompilerInput]: @cached_property def compilation_target_path(self): p = PurePath(self.compiler_data.file_input.resolved_path) - p = os.path.relpath(p) + p = safe_relpath(p) return _anonymize(p) @cached_property @@ -121,7 +120,7 @@ def used_search_paths(self) -> list[str]: sps = [sp for sp, count in tmp.items() if count > 0] assert len(sps) > 0 - return [_anonymize(os.path.relpath(sp)) for sp in sps] + return [_anonymize(safe_relpath(sp)) for sp in sps] class OutputBundleWriter: diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index d05e494b80..90bb631e14 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -1,4 +1,3 @@ -import os from pathlib import Path, PurePath from typing import Any, Optional @@ -58,7 +57,7 @@ from vyper.semantics.types.function import ContractFunctionT from vyper.semantics.types.module import ModuleT from vyper.semantics.types.utils import type_from_annotation -from vyper.utils import OrderedSet +from vyper.utils import OrderedSet, safe_relpath def analyze_module( @@ -921,7 +920,7 @@ def _load_builtin_import(level: int, module_str: str) -> tuple[CompilerInput, In # hygiene: convert to relpath to avoid leaking user directory info # (note Path.relative_to cannot handle absolute to relative path # conversion, so we must use the `os` module). - builtins_path = os.path.relpath(builtins_path) + builtins_path = safe_relpath(builtins_path) search_path = Path(builtins_path).parent.parent.parent # generate an input bundle just because it knows how to build paths. diff --git a/vyper/utils.py b/vyper/utils.py index 3f19a9d15c..5307cd115e 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -4,6 +4,7 @@ import enum import functools import hashlib +import os import sys import time import traceback @@ -599,3 +600,12 @@ def annotate_source_code( cleanup_lines += [""] * (num_lines - len(cleanup_lines)) return "\n".join(cleanup_lines) + + +def safe_relpath(path): + try: + return os.path.relpath(path) + except ValueError: + # on Windows, if path and curdir are on different drives, an exception + # can be thrown + return path From 0e29db0d7f8bd0689c0d30c7de1796001382ec32 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 4 Oct 2024 17:58:37 -0400 Subject: [PATCH 06/13] feat[venom]: store expansion pass (#4068) expand `extract_literals` pass (introduced in bb9129aaa42b2f810f) to also extract variables and rename it to `store_expansion`, allowing for the dft pass to reorder variable uses more effectively. since this also gives us the guarantee that each variable is used exactly once (besides by store instructions), this allows us to make some simplifications in `venom_to_assembly.py`, since we no longer need to account for the same variable occurring on the stack more than one time (cf. for example 5d8280feec16f86ae). this results in a stack scheduler improvement. for example: - examples/tokens/ERC20.vy has a 20 byte codesize improvement - examples/auctions/blind_auction.vy has a 145 byte codesize improvement - examples/voting/ballot.vy has a 28 byte codesize improvement across a range of contracts, the improvement seems to be between 1-2%, but sometimes as high as 5% since stack operands are now guaranteed to be unique, the old rule to avoid swapping if two stack operands are the same no longer works. to address this, this commit adds an equivalence analysis. this creates equivalence sets of equivalent variables based on store chains, and then generalizes the rule from "don't swap if two stack operands are the same" to "don't swap if two stack operands are equivalent". --- .../compiler/venom/test_duplicate_operands.py | 12 +- .../unit/compiler/venom/test_stack_cleanup.py | 3 +- .../unit/compiler/venom/test_stack_reorder.py | 5 + vyper/venom/__init__.py | 5 +- vyper/venom/analysis/equivalent_vars.py | 41 ++++++ ...extract_literals.py => store_expansion.py} | 13 +- vyper/venom/stack_model.py | 7 +- vyper/venom/venom_to_assembly.py | 125 +++++++++--------- 8 files changed, 133 insertions(+), 78 deletions(-) create mode 100644 vyper/venom/analysis/equivalent_vars.py rename vyper/venom/passes/{extract_literals.py => store_expansion.py} (73%) diff --git a/tests/unit/compiler/venom/test_duplicate_operands.py b/tests/unit/compiler/venom/test_duplicate_operands.py index fbff0835d2..ab55649dae 100644 --- a/tests/unit/compiler/venom/test_duplicate_operands.py +++ b/tests/unit/compiler/venom/test_duplicate_operands.py @@ -1,6 +1,8 @@ from vyper.compiler.settings import OptimizationLevel from vyper.venom import generate_assembly_experimental +from vyper.venom.analysis.analysis import IRAnalysesCache from vyper.venom.context import IRContext +from vyper.venom.passes.store_expansion import StoreExpansionPass def test_duplicate_operands(): @@ -13,7 +15,7 @@ def test_duplicate_operands(): %3 = mul %1, %2 stop - Should compile to: [PUSH1, 10, DUP1, DUP1, DUP1, ADD, MUL, POP, STOP] + Should compile to: [PUSH1, 10, DUP1, DUP2, ADD, MUL, POP, STOP] """ ctx = IRContext() fn = ctx.create_function("test") @@ -23,5 +25,9 @@ def test_duplicate_operands(): bb.append_instruction("mul", sum_, op) bb.append_instruction("stop") - asm = generate_assembly_experimental(ctx, optimize=OptimizationLevel.GAS) - assert asm == ["PUSH1", 10, "DUP1", "DUP1", "ADD", "MUL", "POP", "STOP"] + ac = IRAnalysesCache(fn) + StoreExpansionPass(ac, fn).run_pass() + + optimize = OptimizationLevel.GAS + asm = generate_assembly_experimental(ctx, optimize=optimize) + assert asm == ["PUSH1", 10, "DUP1", "DUP2", "ADD", "MUL", "POP", "STOP"] diff --git a/tests/unit/compiler/venom/test_stack_cleanup.py b/tests/unit/compiler/venom/test_stack_cleanup.py index 6015cf1c41..7198861771 100644 --- a/tests/unit/compiler/venom/test_stack_cleanup.py +++ b/tests/unit/compiler/venom/test_stack_cleanup.py @@ -9,7 +9,8 @@ def test_cleanup_stack(): bb = fn.get_basic_block() ret_val = bb.append_instruction("param") op = bb.append_instruction("store", 10) - bb.append_instruction("add", op, op) + op2 = bb.append_instruction("store", op) + bb.append_instruction("add", op, op2) bb.append_instruction("ret", ret_val) asm = generate_assembly_experimental(ctx, optimize=OptimizationLevel.GAS) diff --git a/tests/unit/compiler/venom/test_stack_reorder.py b/tests/unit/compiler/venom/test_stack_reorder.py index a9f505984e..a15dd4d540 100644 --- a/tests/unit/compiler/venom/test_stack_reorder.py +++ b/tests/unit/compiler/venom/test_stack_reorder.py @@ -1,5 +1,7 @@ from vyper.venom import generate_assembly_experimental +from vyper.venom.analysis.analysis import IRAnalysesCache from vyper.venom.context import IRContext +from vyper.venom.passes.store_expansion import StoreExpansionPass def test_stack_reorder(): @@ -25,4 +27,7 @@ def test_stack_reorder(): bb.append_instruction("ret", ret_val) + ac = IRAnalysesCache(fn) + StoreExpansionPass(ac, fn).run_pass() + generate_assembly_experimental(ctx) diff --git a/vyper/venom/__init__.py b/vyper/venom/__init__.py index afd79fc44f..a5f51b787d 100644 --- a/vyper/venom/__init__.py +++ b/vyper/venom/__init__.py @@ -12,13 +12,13 @@ from vyper.venom.passes.algebraic_optimization import AlgebraicOptimizationPass from vyper.venom.passes.branch_optimization import BranchOptimizationPass from vyper.venom.passes.dft import DFTPass -from vyper.venom.passes.extract_literals import ExtractLiteralsPass from vyper.venom.passes.make_ssa import MakeSSA from vyper.venom.passes.mem2var import Mem2Var from vyper.venom.passes.remove_unused_variables import RemoveUnusedVariablesPass from vyper.venom.passes.sccp import SCCP from vyper.venom.passes.simplify_cfg import SimplifyCFGPass from vyper.venom.passes.store_elimination import StoreElimination +from vyper.venom.passes.store_expansion import StoreExpansionPass from vyper.venom.venom_to_assembly import VenomCompiler DEFAULT_OPT_LEVEL = OptimizationLevel.default() @@ -54,8 +54,9 @@ def _run_passes(fn: IRFunction, optimize: OptimizationLevel) -> None: SimplifyCFGPass(ac, fn).run_pass() AlgebraicOptimizationPass(ac, fn).run_pass() BranchOptimizationPass(ac, fn).run_pass() - ExtractLiteralsPass(ac, fn).run_pass() RemoveUnusedVariablesPass(ac, fn).run_pass() + + StoreExpansionPass(ac, fn).run_pass() DFTPass(ac, fn).run_pass() diff --git a/vyper/venom/analysis/equivalent_vars.py b/vyper/venom/analysis/equivalent_vars.py new file mode 100644 index 0000000000..9b0c03e3d1 --- /dev/null +++ b/vyper/venom/analysis/equivalent_vars.py @@ -0,0 +1,41 @@ +from vyper.venom.analysis.analysis import IRAnalysis +from vyper.venom.analysis.dfg import DFGAnalysis +from vyper.venom.basicblock import IRVariable + + +class VarEquivalenceAnalysis(IRAnalysis): + """ + Generate equivalence sets of variables. This is used to avoid swapping + variables which are the same during venom_to_assembly. Theoretically, + the DFTPass should order variable declarations optimally, but, it is + not aware of the "pickaxe" heuristic in venom_to_assembly, so they can + interfere. + """ + + def analyze(self): + dfg = self.analyses_cache.request_analysis(DFGAnalysis) + + equivalence_set: dict[IRVariable, int] = {} + + for bag, (var, inst) in enumerate(dfg._dfg_outputs.items()): + if inst.opcode != "store": + continue + + source = inst.operands[0] + + assert var not in equivalence_set # invariant + if source in equivalence_set: + equivalence_set[var] = equivalence_set[source] + continue + else: + equivalence_set[var] = bag + equivalence_set[source] = bag + + self._equivalence_set = equivalence_set + + def equivalent(self, var1, var2): + if var1 not in self._equivalence_set: + return False + if var2 not in self._equivalence_set: + return False + return self._equivalence_set[var1] == self._equivalence_set[var2] diff --git a/vyper/venom/passes/extract_literals.py b/vyper/venom/passes/store_expansion.py similarity index 73% rename from vyper/venom/passes/extract_literals.py rename to vyper/venom/passes/store_expansion.py index 91c0813e67..7718e67d33 100644 --- a/vyper/venom/passes/extract_literals.py +++ b/vyper/venom/passes/store_expansion.py @@ -1,12 +1,13 @@ from vyper.venom.analysis.dfg import DFGAnalysis from vyper.venom.analysis.liveness import LivenessAnalysis -from vyper.venom.basicblock import IRInstruction, IRLiteral +from vyper.venom.basicblock import IRInstruction, IRLiteral, IRVariable from vyper.venom.passes.base_pass import IRPass -class ExtractLiteralsPass(IRPass): +class StoreExpansionPass(IRPass): """ - This pass extracts literals so that they can be reordered by the DFT pass + This pass extracts literals and variables so that they can be + reordered by the DFT pass """ def run_pass(self): @@ -20,7 +21,7 @@ def _process_bb(self, bb): i = 0 while i < len(bb.instructions): inst = bb.instructions[i] - if inst.opcode in ("store", "offset"): + if inst.opcode in ("store", "offset", "phi", "param"): i += 1 continue @@ -29,9 +30,11 @@ def _process_bb(self, bb): if inst.opcode == "log" and j == 0: continue - if isinstance(op, IRLiteral): + if isinstance(op, (IRVariable, IRLiteral)): var = self.function.get_next_variable() to_insert = IRInstruction("store", [op], var) bb.insert_instruction(to_insert, index=i) inst.operands[j] = var + i += 1 + i += 1 diff --git a/vyper/venom/stack_model.py b/vyper/venom/stack_model.py index a98e5bb25b..e284b41fb2 100644 --- a/vyper/venom/stack_model.py +++ b/vyper/venom/stack_model.py @@ -30,7 +30,7 @@ def push(self, op: IROperand) -> None: def pop(self, num: int = 1) -> None: del self._stack[len(self._stack) - num :] - def get_depth(self, op: IROperand, n: int = 1) -> int: + def get_depth(self, op: IROperand) -> int: """ Returns the depth of the n-th matching operand in the stack map. If the operand is not in the stack map, returns NOT_IN_STACK. @@ -39,10 +39,7 @@ def get_depth(self, op: IROperand, n: int = 1) -> int: for i, stack_op in enumerate(reversed(self._stack)): if stack_op.value == op.value: - if n <= 1: - return -i - else: - n -= 1 + return -i return StackModel.NOT_IN_STACK # type: ignore diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 9de75dab38..56228d53d2 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -1,5 +1,3 @@ -from bisect import insort -from collections import Counter, defaultdict from typing import Any from vyper.exceptions import CompilerPanic, StackTooDeep @@ -13,6 +11,7 @@ ) from vyper.utils import MemoryPositions, OrderedSet from vyper.venom.analysis.analysis import IRAnalysesCache +from vyper.venom.analysis.equivalent_vars import VarEquivalenceAnalysis from vyper.venom.analysis.liveness import LivenessAnalysis from vyper.venom.basicblock import ( IRBasicBlock, @@ -26,6 +25,10 @@ from vyper.venom.passes.normalization import NormalizationPass from vyper.venom.stack_model import StackModel +DEBUG_SHOW_COST = False +if DEBUG_SHOW_COST: + import sys + # instructions which map one-to-one from venom to EVM _ONE_TO_ONE_INSTRUCTIONS = frozenset( [ @@ -153,6 +156,7 @@ def generate_evm(self, no_optimize: bool = False) -> list[str]: NormalizationPass(ac, fn).run_pass() self.liveness_analysis = ac.request_analysis(LivenessAnalysis) + self.equivalence = ac.request_analysis(VarEquivalenceAnalysis) assert fn.normalized, "Non-normalized CFG!" @@ -199,36 +203,19 @@ def generate_evm(self, no_optimize: bool = False) -> list[str]: def _stack_reorder( self, assembly: list, stack: StackModel, stack_ops: list[IROperand], dry_run: bool = False ) -> int: - cost = 0 - if dry_run: assert len(assembly) == 0, "Dry run should not work on assembly" stack = stack.copy() - stack_ops_count = len(stack_ops) - if stack_ops_count == 0: + if len(stack_ops) == 0: return 0 - counts = Counter(stack_ops) - - # positions stores the positions of relevant operands - # on stack for example operand %82 is on positions [0, 3] - # this operand could ocure even more deeper in the stack - # but only those that are needed/relevant in calculation - # are considered - positions: dict[IROperand, list[int]] = defaultdict(list) - for op in stack_ops: - positions[op] = [] - for i in range(counts[op]): - positions[op].append(stack.get_depth(op, i + 1)) - - for i in range(stack_ops_count): - op = stack_ops[i] - final_stack_depth = -(stack_ops_count - i - 1) - depth = positions[op].pop() # type: ignore - assert depth not in range( - -stack_ops_count + 1, final_stack_depth - ), f"{depth} : ({-stack_ops_count - 1}, {final_stack_depth})" + assert len(stack_ops) == len(set(stack_ops)) # precondition + + cost = 0 + for i, op in enumerate(stack_ops): + final_stack_depth = -(len(stack_ops) - i - 1) + depth = stack.get_depth(op) if depth == StackModel.NOT_IN_STACK: raise CompilerPanic(f"Variable {op} not in stack") @@ -236,25 +223,14 @@ def _stack_reorder( if depth == final_stack_depth: continue - if op == stack.peek(final_stack_depth): + to_swap = stack.peek(final_stack_depth) + if self.equivalence.equivalent(op, to_swap): + # perform a "virtual" swap + stack.poke(final_stack_depth, op) + stack.poke(depth, to_swap) continue - # moves the top item to original position - top_item_positions = positions[stack.peek(0)] - if len(top_item_positions) != 0: - top_item_positions.remove(0) - insort(top_item_positions, depth) - cost += self.swap(assembly, stack, depth) - - # moves the item from final position to top - final_item_positions = positions[stack.peek(final_stack_depth)] - if final_stack_depth in final_item_positions: - final_item_positions.remove(final_stack_depth) - final_item_positions.insert(0, 0) - else: - final_item_positions.insert(0, 0) - cost += self.swap(assembly, stack, final_stack_depth) assert stack._stack[-len(stack_ops) :] == stack_ops, (stack, stack_ops) @@ -273,19 +249,14 @@ def _emit_input_operands( # been scheduled to be killed. now it's just a matter of emitting # SWAPs, DUPs and PUSHes until we match the `ops` argument - # dumb heuristic: if the top of stack is not wanted here, swap - # it with something that is wanted - if ops and stack.height > 0 and stack.peek(0) not in ops: - for op in ops: - if isinstance(op, IRVariable) and op not in next_liveness: - self.swap_op(assembly, stack, op) - break + # to validate store expansion invariant - + # each op is emitted at most once. + seen: set[IROperand] = set() - emitted_ops = OrderedSet[IROperand]() for op in ops: if isinstance(op, IRLabel): - # invoke emits the actual instruction itself so we don't need to emit it here - # but we need to add it to the stack map + # invoke emits the actual instruction itself so we don't need + # to emit it here but we need to add it to the stack map if inst.opcode != "invoke": assembly.append(f"_sym_{op.value}") stack.push(op) @@ -300,13 +271,12 @@ def _emit_input_operands( stack.push(op) continue - if op in next_liveness and op not in emitted_ops: + if op in next_liveness: self.dup_op(assembly, stack, op) - if op in emitted_ops: - self.dup_op(assembly, stack, op) - - emitted_ops.add(op) + # guaranteed by store expansion + assert op not in seen, (op, seen) + seen.add(op) def _generate_evm_for_basicblock_r( self, asm: list, basicblock: IRBasicBlock, stack: StackModel @@ -315,6 +285,12 @@ def _generate_evm_for_basicblock_r( return self.visited_basicblocks.add(basicblock) + if DEBUG_SHOW_COST: + print(basicblock, file=sys.stderr) + + ref = asm + asm = [] + # assembly entry point into the block asm.append(f"_sym_{basicblock.label}") asm.append("JUMPDEST") @@ -330,8 +306,14 @@ def _generate_evm_for_basicblock_r( asm.extend(self._generate_evm_for_instruction(inst, stack, next_liveness)) + if DEBUG_SHOW_COST: + print(" ".join(map(str, asm)), file=sys.stderr) + print("\n", file=sys.stderr) + + ref.extend(asm) + for bb in basicblock.reachable: - self._generate_evm_for_basicblock_r(asm, bb, stack.copy()) + self._generate_evm_for_basicblock_r(ref, bb, stack.copy()) # pop values from stack at entry to bb # note this produces the same result(!) no matter which basic block @@ -413,6 +395,7 @@ def _generate_evm_for_instruction( # find an instance of %13 *or* %14 in the stack and replace it with %56. to_be_replaced = stack.peek(depth) if to_be_replaced in next_liveness: + # this branch seems unreachable (maybe due to make_ssa) # %13/%14 is still live(!), so we make a copy of it self.dup(assembly, stack, depth) stack.poke(0, ret) @@ -454,6 +437,13 @@ def _generate_evm_for_instruction( if cost_with_swap > cost_no_swap: operands[-1], operands[-2] = operands[-2], operands[-1] + cost = self._stack_reorder([], stack, operands, dry_run=True) + if DEBUG_SHOW_COST and cost: + print("ENTER", inst, file=sys.stderr) + print(" HAVE", stack, file=sys.stderr) + print(" WANT", operands, file=sys.stderr) + print(" COST", cost, file=sys.stderr) + # final step to get the inputs to this instruction ordered # correctly on the stack self._stack_reorder(assembly, stack, operands) @@ -570,10 +560,21 @@ def _generate_evm_for_instruction( if inst.output not in next_liveness: self.pop(assembly, stack) else: - # peek at next_liveness to find the next scheduled item, - # and optimistically swap with it + # heuristic: peek at next_liveness to find the next scheduled + # item, and optimistically swap with it + if DEBUG_SHOW_COST: + stack0 = stack.copy() + next_scheduled = next_liveness.last() - self.swap_op(assembly, stack, next_scheduled) + cost = 0 + if not self.equivalence.equivalent(inst.output, next_scheduled): + cost = self.swap_op(assembly, stack, next_scheduled) + + if DEBUG_SHOW_COST and cost != 0: + print("ENTER", inst, file=sys.stderr) + print(" HAVE", stack0, file=sys.stderr) + print(" NEXT LIVENESS", next_liveness, file=sys.stderr) + print(" NEW_STACK", stack, file=sys.stderr) return apply_line_numbers(inst, assembly) @@ -595,7 +596,7 @@ def dup(self, assembly, stack, depth): assembly.append(_evm_dup_for(depth)) def swap_op(self, assembly, stack, op): - self.swap(assembly, stack, stack.get_depth(op)) + return self.swap(assembly, stack, stack.get_depth(op)) def dup_op(self, assembly, stack, op): self.dup(assembly, stack, stack.get_depth(op)) From ebe3c0ccfc2a6935dfab10372625d68e8292b4f6 Mon Sep 17 00:00:00 2001 From: z80 <83730246+z80dev@users.noreply.github.com> Date: Fri, 4 Oct 2024 20:39:12 -0400 Subject: [PATCH 07/13] feat[lang]: use keyword arguments for event instantiation (#4257) This commit changes event instantiation from taking positional arguments to taking keyword arguments. For backwards compatibility (and to ease users into v0.4.1), this implementation is fully backwards compatible with the existing positional arguments syntax, but a warning will be emitted if the old syntax is encountered. All unit tests have been updated to use keyword arguments for events, so passing tests give us a lot of confidence the change is correct. In addition, a new unit test was added verifying that positional arguments still compile and work as expected. Misc/refactors: - Factor out `validate_kwargs()` from `StructT._ctor_call_return()` - Rewrite the kwarg validation code for clarity - Add `__sub__` and `__isub__` dunders to `OrderedSet` --------- Co-authored-by: Charles Cooper --- .../functional/builtins/codegen/test_empty.py | 10 +- .../test_default_function.py | 22 ++--- .../features/decorators/test_private.py | 2 +- .../codegen/features/test_logging.py | 99 ++++++++++--------- .../features/test_logging_bytes_extended.py | 16 +-- .../features/test_logging_from_call.py | 16 +-- .../codegen/features/test_memory_dealloc.py | 2 +- .../functional/codegen/modules/test_events.py | 2 +- tests/functional/codegen/types/test_string.py | 2 +- .../test_type_mismatch_exception.py | 2 +- .../syntax/names/test_event_names.py | 12 +-- tests/functional/syntax/test_ann_assign.py | 4 +- .../functional/syntax/test_external_calls.py | 2 +- tests/functional/syntax/test_interfaces.py | 4 +- tests/functional/syntax/test_logging.py | 87 +++++++++++++++- tests/functional/syntax/test_structs.py | 13 ++- vyper/codegen/stmt.py | 8 +- vyper/semantics/analysis/local.py | 12 ++- vyper/semantics/analysis/module.py | 2 +- vyper/semantics/analysis/utils.py | 44 ++++++++- vyper/semantics/types/user.py | 57 +++++------ vyper/utils.py | 10 ++ 22 files changed, 290 insertions(+), 138 deletions(-) diff --git a/tests/functional/builtins/codegen/test_empty.py b/tests/functional/builtins/codegen/test_empty.py index dd6c5c7cc1..3088162238 100644 --- a/tests/functional/builtins/codegen/test_empty.py +++ b/tests/functional/builtins/codegen/test_empty.py @@ -672,11 +672,11 @@ def test_empty_array_in_event_logging(get_contract, get_logs): @external def foo(): log MyLog( - b'hellohellohellohellohellohellohellohellohello', - empty(int128[2][3]), - 314159, - b'helphelphelphelphelphelphelphelphelphelphelp', - empty(uint256[3]) + arg1=b'hellohellohellohellohellohellohellohellohello', + arg2=empty(int128[2][3]), + arg3=314159, + arg4=b'helphelphelphelphelphelphelphelphelphelphelp', + arg5=empty(uint256[3]) ) """ diff --git a/tests/functional/codegen/calling_convention/test_default_function.py b/tests/functional/codegen/calling_convention/test_default_function.py index 4d54e31f91..08d9c08678 100644 --- a/tests/functional/codegen/calling_convention/test_default_function.py +++ b/tests/functional/codegen/calling_convention/test_default_function.py @@ -28,7 +28,7 @@ def test_basic_default(env, get_logs, get_contract): @external @payable def __default__(): - log Sent(msg.sender) + log Sent(sender=msg.sender) """ c = get_contract(code) env.set_balance(env.deployer, 10**18) @@ -46,13 +46,13 @@ def test_basic_default_default_param_function(env, get_logs, get_contract): @external @payable def fooBar(a: int128 = 12345) -> int128: - log Sent(empty(address)) + log Sent(sender=empty(address)) return a @external @payable def __default__(): - log Sent(msg.sender) + log Sent(sender=msg.sender) """ c = get_contract(code) env.set_balance(env.deployer, 10**18) @@ -69,7 +69,7 @@ def test_basic_default_not_payable(env, tx_failed, get_contract): @external def __default__(): - log Sent(msg.sender) + log Sent(sender=msg.sender) """ c = get_contract(code) env.set_balance(env.deployer, 10**17) @@ -103,7 +103,7 @@ def test_always_public_2(assert_compile_failed, get_contract): sender: indexed(address) def __default__(): - log Sent(msg.sender) + log Sent(sender=msg.sender) """ assert_compile_failed(lambda: get_contract(code)) @@ -119,12 +119,12 @@ def test_zero_method_id(env, get_logs, get_contract, tx_failed): @payable # function selector: 0x00000000 def blockHashAskewLimitary(v: uint256) -> uint256: - log Sent(2) + log Sent(sig=2) return 7 @external def __default__(): - log Sent(1) + log Sent(sig=1) """ c = get_contract(code) @@ -165,12 +165,12 @@ def test_another_zero_method_id(env, get_logs, get_contract, tx_failed): @payable # function selector: 0x00000000 def wycpnbqcyf() -> uint256: - log Sent(2) + log Sent(sig=2) return 7 @external def __default__(): - log Sent(1) + log Sent(sig=1) """ c = get_contract(code) @@ -205,12 +205,12 @@ def test_partial_selector_match_trailing_zeroes(env, get_logs, get_contract): @payable # function selector: 0xd88e0b00 def fow() -> uint256: - log Sent(2) + log Sent(sig=2) return 7 @external def __default__(): - log Sent(1) + log Sent(sig=1) """ c = get_contract(code) diff --git a/tests/functional/codegen/features/decorators/test_private.py b/tests/functional/codegen/features/decorators/test_private.py index d313aa3bda..b9e34ea49b 100644 --- a/tests/functional/codegen/features/decorators/test_private.py +++ b/tests/functional/codegen/features/decorators/test_private.py @@ -436,7 +436,7 @@ def i_am_me() -> bool: @external @nonpayable def whoami() -> address: - log Addr(self._whoami()) + log Addr(addr=self._whoami()) return self._whoami() """ diff --git a/tests/functional/codegen/features/test_logging.py b/tests/functional/codegen/features/test_logging.py index cf77a30bd9..2bb646e6ef 100644 --- a/tests/functional/codegen/features/test_logging.py +++ b/tests/functional/codegen/features/test_logging.py @@ -5,13 +5,14 @@ from tests.utils import decimal_to_int from vyper import compile_code from vyper.exceptions import ( - ArgumentException, EventDeclarationException, + InstantiationException, InvalidType, NamespaceCollision, StructureException, TypeMismatch, UndeclaredDefinition, + UnknownAttribute, ) from vyper.utils import keccak256 @@ -50,7 +51,7 @@ def test_event_logging_with_topics(get_logs, keccak, get_contract): @external def foo(): self.a = b"bar" - log MyLog(self.a) + log MyLog(arg1=self.a) """ c = get_contract(loggy_code) @@ -78,7 +79,7 @@ def test_event_logging_with_multiple_topics(env, keccak, get_logs, get_contract) @external def foo(): - log MyLog(-2, True, self) + log MyLog(arg1=-2, arg2=True, arg3=self) """ c = get_contract(loggy_code) @@ -120,7 +121,7 @@ def test_event_logging_with_multiple_topics_var_and_store(get_contract, get_logs def foo(arg1: int128): a: bool = True self.b = self - log MyLog(arg1, a, self.b) + log MyLog(arg1=arg1, arg2=a, arg3=self.b) """ c = get_contract(code) @@ -141,13 +142,13 @@ def test_logging_the_same_event_multiple_times_with_topics(env, keccak, get_logs @external def foo(): - log MyLog(1, self) - log MyLog(1, self) + log MyLog(arg1=1, arg2=self) + log MyLog(arg1=1, arg2=self) @external def bar(): - log MyLog(1, self) - log MyLog(1, self) + log MyLog(arg1=1, arg2=self) + log MyLog(arg1=1, arg2=self) """ c = get_contract(loggy_code) @@ -198,7 +199,7 @@ def test_event_logging_with_data(get_logs, keccak, get_contract): @external def foo(): - log MyLog(123) + log MyLog(arg1=123) """ c = get_contract(loggy_code) @@ -231,8 +232,16 @@ def test_event_logging_with_fixed_array_data(env, keccak, get_logs, get_contract @external def foo(): - log MyLog([1,2], [block.timestamp, block.timestamp+1, block.timestamp+2], [[1,2],[1,2]]) - log MyLog([1,2], [block.timestamp, block.timestamp+1, block.timestamp+2], [[1,2],[1,2]]) + log MyLog( + arg1=[1,2], + arg2=[block.timestamp, block.timestamp+1, block.timestamp+2], + arg3=[[1,2],[1,2]] + ) + log MyLog( + arg1=[1,2], + arg2=[block.timestamp, block.timestamp+1, block.timestamp+2], + arg3=[[1,2],[1,2]] + ) """ c = get_contract(loggy_code) @@ -271,7 +280,7 @@ def test_logging_with_input_bytes_1(env, keccak, get_logs, get_contract): @external def foo(arg1: Bytes[29], arg2: Bytes[31]): - log MyLog(b'bar', arg1, arg2) + log MyLog(arg1=b'bar', arg2=arg1, arg3=arg2) """ c = get_contract(loggy_code) @@ -307,7 +316,7 @@ def test_event_logging_with_bytes_input_2(env, keccak, get_logs, get_contract): @external def foo(_arg1: Bytes[20]): - log MyLog(_arg1) + log MyLog(arg1=_arg1) """ c = get_contract(loggy_code) @@ -335,7 +344,7 @@ def test_event_logging_with_bytes_input_3(get_logs, keccak, get_contract): @external def foo(_arg1: Bytes[5]): - log MyLog(_arg1) + log MyLog(arg1=_arg1) """ c = get_contract(loggy_code) @@ -369,7 +378,7 @@ def test_event_logging_with_data_with_different_types(env, keccak, get_logs, get @external def foo(): - log MyLog(123, b'home', b'bar', 0xc305c901078781C232A2a521C2aF7980f8385ee9, self, block.timestamp) # noqa: E501 + log MyLog(arg1=123, arg2=b'home', arg3=b'bar', arg4=0xc305c901078781C232A2a521C2aF7980f8385ee9, arg5=self, arg6=block.timestamp) # noqa: E501 """ c = get_contract(loggy_code) @@ -412,7 +421,7 @@ def test_event_logging_with_topics_and_data_1(env, keccak, get_logs, get_contrac @external def foo(): - log MyLog(1, b'bar') + log MyLog(arg1=1, arg2=b'bar') """ c = get_contract(loggy_code) @@ -457,8 +466,8 @@ def test_event_logging_with_multiple_logs_topics_and_data(env, keccak, get_logs, @external def foo(): - log MyLog(1, b'bar') - log YourLog(self, MyStruct(x=1, y=b'abc', z=SmallStruct(t='house', w=13.5))) + log MyLog(arg1=1, arg2=b'bar') + log YourLog(arg1=self, arg2=MyStruct(x=1, y=b'abc', z=SmallStruct(t='house', w=13.5))) """ c = get_contract(loggy_code) @@ -524,7 +533,7 @@ def test_fails_when_input_is_the_wrong_type(tx_failed, get_contract): @external def foo_(): - log MyLog(b'yo') + log MyLog(arg1=b'yo') """ with tx_failed(TypeMismatch): @@ -539,7 +548,7 @@ def test_fails_when_topic_is_the_wrong_size(tx_failed, get_contract): @external def foo(): - log MyLog(b'bars') + log MyLog(arg1=b'bars') """ with tx_failed(TypeMismatch): @@ -553,7 +562,7 @@ def test_fails_when_input_topic_is_the_wrong_size(tx_failed, get_contract): @external def foo(arg1: Bytes[4]): - log MyLog(arg1) + log MyLog(arg1=arg1) """ with tx_failed(TypeMismatch): @@ -567,7 +576,7 @@ def test_fails_when_data_is_the_wrong_size(tx_failed, get_contract): @external def foo(): - log MyLog(b'bars') + log MyLog(arg1=b'bars') """ with tx_failed(TypeMismatch): @@ -581,7 +590,7 @@ def test_fails_when_input_data_is_the_wrong_size(tx_failed, get_contract): @external def foo(arg1: Bytes[4]): - log MyLog(arg1) + log MyLog(arg1=arg1) """ with tx_failed(TypeMismatch): @@ -610,7 +619,7 @@ def test_logging_fails_with_over_three_topics(tx_failed, get_contract): @deploy def __init__(): - log MyLog(1, 2, 3, 4) + log MyLog(arg1=1, arg2=2, arg3=3, arg4=4) """ with tx_failed(EventDeclarationException): @@ -650,7 +659,7 @@ def test_logging_fails_with_topic_type_mismatch(tx_failed, get_contract): @external def foo(): - log MyLog(self) + log MyLog(arg1=self) """ with tx_failed(TypeMismatch): @@ -664,7 +673,7 @@ def test_logging_fails_with_data_type_mismatch(tx_failed, get_contract): @external def foo(): - log MyLog(self) + log MyLog(arg1=self) """ with tx_failed(TypeMismatch): @@ -680,9 +689,9 @@ def test_logging_fails_when_number_of_arguments_is_greater_than_declaration( @external def foo(): - log MyLog(1, 2) + log MyLog(arg1=1, arg2=2) """ - with tx_failed(ArgumentException): + with tx_failed(UnknownAttribute): get_contract(loggy_code) @@ -694,9 +703,9 @@ def test_logging_fails_when_number_of_arguments_is_less_than_declaration(tx_fail @external def foo(): - log MyLog(1) + log MyLog(arg1=1) """ - with tx_failed(ArgumentException): + with tx_failed(InstantiationException): get_contract(loggy_code) @@ -852,7 +861,7 @@ def test_variable_list_packing(get_logs, get_contract): @external def foo(): a: int128[4] = [1, 2, 3, 4] - log Bar(a) + log Bar(_value=a) """ c = get_contract(code) @@ -868,7 +877,7 @@ def test_literal_list_packing(get_logs, get_contract): @external def foo(): - log Bar([1, 2, 3, 4]) + log Bar(_value=[1, 2, 3, 4]) """ c = get_contract(code) @@ -886,7 +895,7 @@ def test_storage_list_packing(get_logs, get_contract): @external def foo(): - log Bar(self.x) + log Bar(_value=self.x) @external def set_list(): @@ -910,7 +919,7 @@ def test_passed_list_packing(get_logs, get_contract): @external def foo(barbaric: int128[4]): - log Bar(barbaric) + log Bar(_value=barbaric) """ c = get_contract(code) @@ -926,7 +935,7 @@ def test_variable_decimal_list_packing(get_logs, get_contract): @external def foo(): - log Bar([1.11, 2.22, 3.33, 4.44]) + log Bar(_value=[1.11, 2.22, 3.33, 4.44]) """ c = get_contract(code) @@ -949,7 +958,7 @@ def test_storage_byte_packing(get_logs, get_contract): @external def foo(a: int128): - log MyLog(self.x) + log MyLog(arg1=self.x) @external def setbytez(): @@ -975,7 +984,7 @@ def test_storage_decimal_list_packing(get_logs, get_contract): @external def foo(): - log Bar(self.x) + log Bar(_value=self.x) @external def set_list(): @@ -1004,7 +1013,7 @@ def test_logging_fails_when_input_is_too_big(tx_failed, get_contract): @external def foo(inp: Bytes[33]): - log Bar(inp) + log Bar(_value=inp) """ with tx_failed(TypeMismatch): get_contract(code) @@ -1019,7 +1028,7 @@ def test_2nd_var_list_packing(get_logs, get_contract): @external def foo(): a: int128[4] = [1, 2, 3, 4] - log Bar(10, a) + log Bar(arg1=10, arg2=a) """ c = get_contract(code) @@ -1037,7 +1046,7 @@ def test_2nd_var_storage_list_packing(get_logs, get_contract): @external def foo(): - log Bar(10, self.x) + log Bar(arg1=10, arg2=self.x) @external def set_list(): @@ -1071,7 +1080,7 @@ def __init__(): @external def foo(): v: int128[3] = [7, 8, 9] - log Bar(10, self.x, b"test", v, self.y) + log Bar(arg1=10, arg2=self.x, arg3=b"test", arg4=v, arg5=self.y) @external def set_list(): @@ -1104,7 +1113,7 @@ def test_hashed_indexed_topics_calldata(get_logs, keccak, get_contract): @external def foo(a: Bytes[36], b: int128, c: String[7]): - log MyLog(a, b, c) + log MyLog(arg1=a, arg2=b, arg3=c) """ c = get_contract(loggy_code) @@ -1144,7 +1153,7 @@ def foo(): a: Bytes[10] = b"potato" b: int128 = -777 c: String[44] = "why hello, neighbor! how are you today?" - log MyLog(a, b, c) + log MyLog(arg1=a, arg2=b, arg3=c) """ c = get_contract(loggy_code) @@ -1191,7 +1200,7 @@ def setter(_a: Bytes[32], _b: int128, _c: String[6]): @external def foo(): - log MyLog(self.a, self.b, self.c) + log MyLog(arg1=self.a, arg2=self.b, arg3=self.c) """ c = get_contract(loggy_code) @@ -1229,7 +1238,7 @@ def test_hashed_indexed_topics_storxxage(get_logs, keccak, get_contract): @external def foo(): - log MyLog(b"wow", 666, "madness!") + log MyLog(arg1=b"wow", arg2=666, arg3="madness!") """ c = get_contract(loggy_code) diff --git a/tests/functional/codegen/features/test_logging_bytes_extended.py b/tests/functional/codegen/features/test_logging_bytes_extended.py index 6b84cdd23a..64c848bb8e 100644 --- a/tests/functional/codegen/features/test_logging_bytes_extended.py +++ b/tests/functional/codegen/features/test_logging_bytes_extended.py @@ -7,7 +7,7 @@ def test_bytes_logging_extended(get_contract, get_logs): @external def foo(): - log MyLog(667788, b'hellohellohellohellohellohellohellohellohello', 334455) + log MyLog(arg1=667788, arg2=b'hellohellohellohellohellohellohellohellohello', arg3=334455) """ c = get_contract(code) @@ -31,7 +31,7 @@ def foo(): a: Bytes[64] = b'hellohellohellohellohellohellohellohellohello' b: Bytes[64] = b'hellohellohellohellohellohellohellohello' # test literal much smaller than buffer - log MyLog(a, b, b'hello') + log MyLog(arg1=a, arg2=b, arg3=b'hello') """ c = get_contract(code) @@ -51,7 +51,7 @@ def test_bytes_logging_extended_passthrough(get_contract, get_logs): @external def foo(a: int128, b: Bytes[64], c: int128): - log MyLog(a, b, c) + log MyLog(arg1=a, arg2=b, arg3=c) """ c = get_contract(code) @@ -77,7 +77,7 @@ def test_bytes_logging_extended_storage(get_contract, get_logs): @external def foo(): - log MyLog(self.a, self.b, self.c) + log MyLog(arg1=self.a, arg2=self.b, arg3=self.c) @external def set(x: int128, y: Bytes[64], z: int128): @@ -114,10 +114,10 @@ def test_bytes_logging_extended_mixed_with_lists(get_contract, get_logs): @external def foo(): log MyLog( - [[24, 26], [12, 10]], - b'hellohellohellohellohellohellohellohellohello', - 314159, - b'helphelphelphelphelphelphelphelphelphelphelp' + arg1=[[24, 26], [12, 10]], + arg2=b'hellohellohellohellohellohellohellohellohello', + arg3=314159, + arg4=b'helphelphelphelphelphelphelphelphelphelphelp' ) """ diff --git a/tests/functional/codegen/features/test_logging_from_call.py b/tests/functional/codegen/features/test_logging_from_call.py index 190be7b4f4..2b14cd8398 100644 --- a/tests/functional/codegen/features/test_logging_from_call.py +++ b/tests/functional/codegen/features/test_logging_from_call.py @@ -21,11 +21,11 @@ def to_bytes32(_value: uint256) -> bytes32: @external def test_func(_value: uint256): data2: Bytes[60] = concat(self.to_bytes32(_value),self.to_bytes(_value),b"testing") - log TestLog(self.to_bytes32(_value), data2, self.to_bytes(_value)) + log TestLog(testData1=self.to_bytes32(_value), testData2=data2, testData3=self.to_bytes(_value)) loggedValue: bytes32 = self.to_bytes32(_value) loggedValue2: Bytes[8] = self.to_bytes(_value) - log TestLog(loggedValue, data2, loggedValue2) + log TestLog(testData1=loggedValue, testData2=data2, testData3=loggedValue2) """ c = get_contract(code) @@ -65,8 +65,8 @@ def test_func(_value: uint256,input: Bytes[133]): data2: Bytes[200] = b"hello world" - # log TestLog(self.to_bytes32(_value),input,self.to_bytes(_value)) - log TestLog(self.to_bytes32(_value),input,"bababa") + # log TestLog(testData1=self.to_bytes32(_value),testData2=input,testData3=self.to_bytes(_value)) + log TestLog(testData1=self.to_bytes32(_value),testData2=input,testData3="bababa") """ c = get_contract(code) @@ -99,8 +99,8 @@ def test_func(_value: uint256,input: Bytes[133]): data2: Bytes[200] = b"hello world" # log will be malformed - # log TestLog(self.to_bytes32(_value),input,self.to_bytes(_value)) - log TestLog(self.to_bytes32(_value), input) + # log TestLog(testData1=self.to_bytes32(_value),testData2=input,testData3=self.to_bytes(_value)) + log TestLog(testData1=self.to_bytes32(_value), testData2=input) """ c = get_contract(code) @@ -137,12 +137,12 @@ def test_func(_value: uint256,input: Bytes[2048]): data2: Bytes[2064] = concat(self.to_bytes(_value),self.to_bytes(_value),input) # log will be malformed - log TestLog(self.to_bytes32(_value), data2, self.to_bytes(_value)) + log TestLog(testData1=self.to_bytes32(_value), testData2=data2, testData3=self.to_bytes(_value)) loggedValue: Bytes[8] = self.to_bytes(_value) # log will be normal - log TestLog(self.to_bytes32(_value),data2,loggedValue) + log TestLog(testData1=self.to_bytes32(_value),testData2=data2,testData3=loggedValue) """ c = get_contract(code) diff --git a/tests/functional/codegen/features/test_memory_dealloc.py b/tests/functional/codegen/features/test_memory_dealloc.py index 3be57038ef..b733de736b 100644 --- a/tests/functional/codegen/features/test_memory_dealloc.py +++ b/tests/functional/codegen/features/test_memory_dealloc.py @@ -9,7 +9,7 @@ def sendit(): nonpayable @external def foo(target: address) -> uint256[2]: - log Shimmy(empty(address), 3) + log Shimmy(a=empty(address), b=3) amount: uint256 = 1 flargen: uint256 = 42 extcall Other(target).sendit() diff --git a/tests/functional/codegen/modules/test_events.py b/tests/functional/codegen/modules/test_events.py index ae5198cf90..c32a66caec 100644 --- a/tests/functional/codegen/modules/test_events.py +++ b/tests/functional/codegen/modules/test_events.py @@ -50,7 +50,7 @@ def test_module_event_indexed(get_contract, make_input_bundle, get_logs): @internal def foo(): - log MyEvent(5, 6) + log MyEvent(x=5, y=6) """ main = """ import lib1 diff --git a/tests/functional/codegen/types/test_string.py b/tests/functional/codegen/types/test_string.py index 51899b50f3..1c186eeb6e 100644 --- a/tests/functional/codegen/types/test_string.py +++ b/tests/functional/codegen/types/test_string.py @@ -116,7 +116,7 @@ def test_logging_extended_string(get_contract, get_logs): @external def foo(): - log MyLog(667788, 'hellohellohellohellohellohellohellohellohello', 334455) + log MyLog(arg1=667788, arg2='hellohellohellohellohellohellohellohellohello', arg3=334455) """ c = get_contract(code) diff --git a/tests/functional/syntax/exceptions/test_type_mismatch_exception.py b/tests/functional/syntax/exceptions/test_type_mismatch_exception.py index 514f2df618..76c5c481f0 100644 --- a/tests/functional/syntax/exceptions/test_type_mismatch_exception.py +++ b/tests/functional/syntax/exceptions/test_type_mismatch_exception.py @@ -41,7 +41,7 @@ def foo(): message: String[1] @external def foo(): - log Foo("abcd") + log Foo(message="abcd") """, # Address literal must be checksummed """ diff --git a/tests/functional/syntax/names/test_event_names.py b/tests/functional/syntax/names/test_event_names.py index 367b646bfe..28cd6bdad0 100644 --- a/tests/functional/syntax/names/test_event_names.py +++ b/tests/functional/syntax/names/test_event_names.py @@ -26,7 +26,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: temp_var : int128 = i - log int128(temp_var) + log int128(variable=temp_var) return temp_var """, NamespaceCollision, @@ -39,7 +39,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: temp_var : int128 = i - log decimal(temp_var) + log decimal(variable=temp_var) return temp_var """, NamespaceCollision, @@ -52,7 +52,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: temp_var : int128 = i - log wei(temp_var) + log wei(variable=temp_var) return temp_var """, StructureException, @@ -65,7 +65,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: temp_var : int128 = i - log false(temp_var) + log false(variable=temp_var) return temp_var """, StructureException, @@ -102,7 +102,7 @@ def test_varname_validity_fail(bad_code, exc): @external def foo(i: int128) -> int128: variable : int128 = i - log Assigned(variable) + log Assigned(variable=variable) return variable """, """ @@ -122,7 +122,7 @@ def foo(i: int128) -> int128: @external def foo(i: int128) -> int128: variable : int128 = i - log Assigned1(variable) + log Assigned1(variable=variable) return variable """, ] diff --git a/tests/functional/syntax/test_ann_assign.py b/tests/functional/syntax/test_ann_assign.py index 23ebeb9560..fba9eff38d 100644 --- a/tests/functional/syntax/test_ann_assign.py +++ b/tests/functional/syntax/test_ann_assign.py @@ -3,11 +3,11 @@ from vyper import compiler from vyper.exceptions import ( + InstantiationException, InvalidAttribute, TypeMismatch, UndeclaredDefinition, UnknownAttribute, - VariableDeclarationException, ) fail_list = [ @@ -73,7 +73,7 @@ def foo() -> int128: def foo() -> int128: s: S = S(a=1) """, - VariableDeclarationException, + InstantiationException, ), ( """ diff --git a/tests/functional/syntax/test_external_calls.py b/tests/functional/syntax/test_external_calls.py index a8fb5ae87b..fd6fa28cc9 100644 --- a/tests/functional/syntax/test_external_calls.py +++ b/tests/functional/syntax/test_external_calls.py @@ -61,7 +61,7 @@ def foo(f: Foo): s: uint256 = staticcall f.foo() """, # TODO: tokenizer currently has issue with log+staticcall/extcall, e.g. - # `log Bar(staticcall f.foo() + extcall f.bar())` + # `log Bar(_value=staticcall f.foo() + extcall f.bar())` ] diff --git a/tests/functional/syntax/test_interfaces.py b/tests/functional/syntax/test_interfaces.py index ea06e0ab2f..20813c48d1 100644 --- a/tests/functional/syntax/test_interfaces.py +++ b/tests/functional/syntax/test_interfaces.py @@ -158,12 +158,12 @@ def f(a: uint256): # visibility is nonpayable instead of view @external def transfer(_to : address, _value : uint256) -> bool: - log Transfer(msg.sender, _to, _value) + log Transfer(sender=msg.sender, receiver=_to, value=_value) return True @external def transferFrom(_from : address, _to : address, _value : uint256) -> bool: - log IERC20.Transfer(_from, _to, _value) + log IERC20.Transfer(sender=_from, receiver=_to, value=_value) return True @external diff --git a/tests/functional/syntax/test_logging.py b/tests/functional/syntax/test_logging.py index b96700a128..7f8f141b99 100644 --- a/tests/functional/syntax/test_logging.py +++ b/tests/functional/syntax/test_logging.py @@ -1,7 +1,13 @@ import pytest from vyper import compiler -from vyper.exceptions import StructureException, TypeMismatch +from vyper.exceptions import ( + InstantiationException, + InvalidAttribute, + StructureException, + TypeMismatch, + UnknownAttribute, +) fail_list = [ """ @@ -12,7 +18,7 @@ @external def foo(): - log Bar(self.x) + log Bar(_value=self.x) """, """ event Bar: @@ -21,7 +27,7 @@ def foo(): @external def foo(): x: decimal[4] = [0.0, 0.0, 0.0, 0.0] - log Bar(x) + log Bar(_value=x) """, """ struct Foo: @@ -37,7 +43,7 @@ def foo(): @external def test(): - log Test(-7) + log Test(n=-7) """, ] @@ -48,6 +54,61 @@ def test_logging_fail(bad_code): compiler.compile_code(bad_code) +def test_logging_fail_mixed_positional_kwargs(): + code = """ +event Test: + n: uint256 + o: uint256 + +@external +def test(): + log Test(7, o=12) + """ + with pytest.raises(InstantiationException): + compiler.compile_code(code) + + +def test_logging_fail_unknown_kwarg(): + code = """ +event Test: + n: uint256 + +@external +def test(): + log Test(n=7, o=12) + """ + with pytest.raises(UnknownAttribute): + compiler.compile_code(code) + + +def test_logging_fail_missing_kwarg(): + code = """ +event Test: + n: uint256 + o: uint256 + +@external +def test(): + log Test(n=7) + """ + with pytest.raises(InstantiationException): + compiler.compile_code(code) + + +def test_logging_fail_kwargs_out_of_order(): + code = """ +event Test: + n: uint256 + o: uint256 + +@external +def test(): + log Test(o=12, n=7) + """ + with pytest.raises(InvalidAttribute): + compiler.compile_code(code) + + @pytest.mark.parametrize("mutability", ["@pure", "@view"]) @pytest.mark.parametrize("visibility", ["@internal", "@external"]) def test_logging_from_non_mutable(mutability, visibility): @@ -58,7 +119,23 @@ def test_logging_from_non_mutable(mutability, visibility): {visibility} {mutability} def test(): - log Test(1) + log Test(n=1) """ with pytest.raises(StructureException): compiler.compile_code(code) + + +def test_logging_with_positional_args(get_contract, get_logs): + # TODO: Remove when positional arguments are fully deprecated + code = """ +event Test: + n: uint256 + +@external +def test(): + log Test(1) + """ + c = get_contract(code) + c.test() + (log,) = get_logs(c, "Test") + assert log.args.n == 1 diff --git a/tests/functional/syntax/test_structs.py b/tests/functional/syntax/test_structs.py index 9a9a397c48..c08859cd92 100644 --- a/tests/functional/syntax/test_structs.py +++ b/tests/functional/syntax/test_structs.py @@ -5,6 +5,7 @@ from vyper import compiler from vyper.exceptions import ( InstantiationException, + InvalidAttribute, StructureException, SyntaxException, TypeMismatch, @@ -32,7 +33,8 @@ def foo(): """, UnknownAttribute, ), - """ + ( + """ struct A: x: int128 y: int128 @@ -41,6 +43,8 @@ def foo(): def foo(): self.a = A(x=1) """, + InstantiationException, + ), """ struct A: x: int128 @@ -61,7 +65,8 @@ def foo(): def foo(): self.a = A(self.b) """, - """ + ( + """ struct A: x: int128 y: int128 @@ -70,6 +75,8 @@ def foo(): def foo(): self.a = A({x: 1}) """, + InstantiationException, + ), """ struct C: c: int128 @@ -386,7 +393,7 @@ def foo(): def foo(): self.b = B(foo=1, foo=2) """, - UnknownAttribute, + InvalidAttribute, ), ( """ diff --git a/vyper/codegen/stmt.py b/vyper/codegen/stmt.py index 225cede747..b1e26d7d5f 100644 --- a/vyper/codegen/stmt.py +++ b/vyper/codegen/stmt.py @@ -93,7 +93,13 @@ def parse_If(self): def parse_Log(self): event = self.stmt._metadata["type"] - args = [Expr(arg, self.context).ir_node for arg in self.stmt.value.args] + if len(self.stmt.value.keywords) > 0: + # keyword arguments + to_compile = [arg.value for arg in self.stmt.value.keywords] + else: + # positional arguments + to_compile = self.stmt.value.args + args = [Expr(arg, self.context).ir_node for arg in to_compile] topic_ir = [] data_ir = [] diff --git a/vyper/semantics/analysis/local.py b/vyper/semantics/analysis/local.py index b5292b1dad..809c6532c6 100644 --- a/vyper/semantics/analysis/local.py +++ b/vyper/semantics/analysis/local.py @@ -810,13 +810,17 @@ def visit_Call(self, node: vy_ast.Call, typ: VyperType) -> None: self.visit(kwarg.value, typ) elif is_type_t(func_type, EventT): - # events have no kwargs + # event ctors expected_types = func_type.typedef.arguments.values() # type: ignore - for arg, typ in zip(node.args, expected_types): - self.visit(arg, typ) + # Handle keyword args if present, otherwise use positional args + if len(node.keywords) > 0: + for kwarg, arg_type in zip(node.keywords, expected_types): + self.visit(kwarg.value, arg_type) + else: + for arg, typ in zip(node.args, expected_types): + self.visit(arg, typ) elif is_type_t(func_type, StructT): # struct ctors - # ctors have no kwargs expected_types = func_type.typedef.members.values() # type: ignore for kwarg, arg_type in zip(node.keywords, expected_types): self.visit(kwarg.value, arg_type) diff --git a/vyper/semantics/analysis/module.py b/vyper/semantics/analysis/module.py index 90bb631e14..6816fbed98 100644 --- a/vyper/semantics/analysis/module.py +++ b/vyper/semantics/analysis/module.py @@ -897,7 +897,7 @@ def _import_to_path(level: int, module_str: str) -> PurePath: base_path = "../" * (level - 1) elif level == 1: base_path = "./" - return PurePath(f"{base_path}{module_str.replace('.','/')}/") + return PurePath(f"{base_path}{module_str.replace('.', '/')}/") # can add more, e.g. "vyper.builtins.interfaces", etc. diff --git a/vyper/semantics/analysis/utils.py b/vyper/semantics/analysis/utils.py index d30eee79e0..9734087fc3 100644 --- a/vyper/semantics/analysis/utils.py +++ b/vyper/semantics/analysis/utils.py @@ -1,9 +1,11 @@ import itertools -from typing import Callable, Iterable, List +from typing import Any, Callable, Iterable, List from vyper import ast as vy_ast from vyper.exceptions import ( CompilerPanic, + InstantiationException, + InvalidAttribute, InvalidLiteral, InvalidOperation, InvalidReference, @@ -694,3 +696,43 @@ def get_expr_writes(node: vy_ast.VyperNode) -> OrderedSet[VarAccess]: ret |= get_expr_writes(c) node._metadata["writes_r"] = ret return ret + + +def validate_kwargs(node: vy_ast.Call, members: dict[str, VyperType], typeclass: str): + # manually validate kwargs for better error messages instead of + # relying on `validate_call_args` + + seen: dict[str, vy_ast.keyword] = {} + membernames = list(members.keys()) + + # check duplicate kwargs + for i, kwarg in enumerate(node.keywords): + # x=5 => kwarg(arg="x", value=Int(5)) + argname = kwarg.arg + if argname in seen: + prev = seen[argname] + raise InvalidAttribute(f"Duplicate {typeclass} argument", prev, kwarg) + seen[argname] = kwarg + + hint: Any # mypy kludge + if argname not in members: + hint = get_levenshtein_error_suggestions(argname, members, 1.0) + raise UnknownAttribute(f"Unknown {typeclass} argument.", kwarg, hint=hint) + + expect_name = membernames[i] + if argname != expect_name: + # out of order key + msg = f"{typeclass} keys are required to be in order, but got" + msg += f" `{argname}` instead of `{expect_name}`." + hint = "as a reminder, the order of the keys in this" + hint += f" {typeclass} are {list(members)}" + raise InvalidAttribute(msg, kwarg, hint=hint) + + expected_type = members[argname] + validate_expected_type(kwarg.value, expected_type) + + missing = OrderedSet(members.keys()) - OrderedSet(seen.keys()) + if len(missing) > 0: + msg = f"{typeclass} instantiation missing fields:" + msg += f" {', '.join(list(missing))}" + raise InstantiationException(msg, node) diff --git a/vyper/semantics/types/user.py b/vyper/semantics/types/user.py index ca8e99bc92..73fa4878c7 100644 --- a/vyper/semantics/types/user.py +++ b/vyper/semantics/types/user.py @@ -7,21 +7,23 @@ from vyper.exceptions import ( EventDeclarationException, FlagDeclarationException, - InvalidAttribute, + InstantiationException, NamespaceCollision, StructureException, UnfoldableNode, - UnknownAttribute, VariableDeclarationException, ) from vyper.semantics.analysis.base import Modifiability -from vyper.semantics.analysis.levenshtein_utils import get_levenshtein_error_suggestions -from vyper.semantics.analysis.utils import check_modifiability, validate_expected_type +from vyper.semantics.analysis.utils import ( + check_modifiability, + validate_expected_type, + validate_kwargs, +) from vyper.semantics.data_locations import DataLocation from vyper.semantics.types.base import VyperType from vyper.semantics.types.subscriptable import HashMapT from vyper.semantics.types.utils import type_from_abi, type_from_annotation -from vyper.utils import keccak256 +from vyper.utils import keccak256, vyper_warn # user defined type @@ -281,6 +283,25 @@ def from_EventDef(cls, base_node: vy_ast.EventDef) -> "EventT": return cls(base_node.name, members, indexed, base_node) def _ctor_call_return(self, node: vy_ast.Call) -> None: + # validate keyword arguments if provided + if len(node.keywords) > 0: + if len(node.args) > 0: + raise InstantiationException( + "Event instantiation requires either all keyword arguments " + "or all positional arguments", + node, + ) + + return validate_kwargs(node, self.arguments, self.typeclass) + + # warn about positional argument depreciation + msg = "Instantiating events with positional arguments is " + msg += "deprecated as of v0.4.1 and will be disallowed " + msg += "in a future release. Use kwargs instead eg. " + msg += "Foo(a=1, b=2)" + + vyper_warn(msg, node) + validate_call_args(node, len(self.arguments)) for arg, expected in zip(node.args, self.arguments.values()): validate_expected_type(arg, expected) @@ -415,31 +436,7 @@ def _ctor_call_return(self, node: vy_ast.Call) -> "StructT": "Struct contains a mapping and so cannot be declared as a literal", node ) - # manually validate kwargs for better error messages instead of - # relying on `validate_call_args` - members = self.member_types.copy() - keys = list(self.member_types.keys()) - for i, kwarg in enumerate(node.keywords): - # x=5 => kwarg(arg="x", value=Int(5)) - argname = kwarg.arg - if argname not in members: - hint = get_levenshtein_error_suggestions(argname, members, 1.0) - raise UnknownAttribute("Unknown or duplicate struct member.", kwarg, hint=hint) - expected = keys[i] - if argname != expected: - raise InvalidAttribute( - "Struct keys are required to be in order, but got " - f"`{argname}` instead of `{expected}`. (Reminder: the " - f"keys in this struct are {list(self.member_types.items())})", - kwarg, - ) - expected_type = members.pop(argname) - validate_expected_type(kwarg.value, expected_type) - - if members: - raise VariableDeclarationException( - f"Struct declaration does not define all fields: {', '.join(list(members))}", node - ) + validate_kwargs(node, self.member_types, self.typeclass) return self diff --git a/vyper/utils.py b/vyper/utils.py index 5307cd115e..d635c78383 100644 --- a/vyper/utils.py +++ b/vyper/utils.py @@ -89,6 +89,7 @@ def update(self, other): def union(self, other): return self | other + # set dunders def __ior__(self, other): self.update(other) return self @@ -101,6 +102,15 @@ def __or__(self, other): def __eq__(self, other): return self._data == other._data + def __isub__(self, other): + self.dropmany(other) + return self + + def __sub__(self, other): + ret = self.copy() + ret.dropmany(other) + return ret + def copy(self): cls = self.__class__ ret = cls.__new__(cls) From 02f8654c5471334e7ee6f3008353dee1a88860a6 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 4 Oct 2024 20:44:25 -0400 Subject: [PATCH 08/13] feat[venom]: add effects to instructions (#4264) this commit adds an `effects.py` file to venom, which describes the effects of opcodes. this is useful for several ongoing efforts, including CSE elimination and DFT pass improvements. --------- Co-authored-by: Harry Kalogirou Co-authored-by: HodanPlodky <36966616+HodanPlodky@users.noreply.github.com> --- vyper/venom/basicblock.py | 7 ++++ vyper/venom/effects.py | 85 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 vyper/venom/effects.py diff --git a/vyper/venom/basicblock.py b/vyper/venom/basicblock.py index 45db8b232f..799dcfb33b 100644 --- a/vyper/venom/basicblock.py +++ b/vyper/venom/basicblock.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING, Any, Iterator, Optional, Union +import vyper.venom.effects as effects from vyper.codegen.ir_node import IRnode from vyper.utils import OrderedSet @@ -238,6 +239,12 @@ def is_volatile(self) -> bool: def is_bb_terminator(self) -> bool: return self.opcode in BB_TERMINATORS + def get_read_effects(self): + return effects.reads.get(self.opcode, effects.EMPTY) + + def get_write_effects(self): + return effects.writes.get(self.opcode, effects.EMPTY) + def get_label_operands(self) -> Iterator[IRLabel]: """ Get all labels in instruction. diff --git a/vyper/venom/effects.py b/vyper/venom/effects.py new file mode 100644 index 0000000000..20cc0e4b02 --- /dev/null +++ b/vyper/venom/effects.py @@ -0,0 +1,85 @@ +from enum import Flag, auto + + +class Effects(Flag): + STORAGE = auto() + TRANSIENT = auto() + MEMORY = auto() + MSIZE = auto() + IMMUTABLES = auto() + RETURNDATA = auto() + LOG = auto() + BALANCE = auto() + EXTCODE = auto() + + +EMPTY = Effects(0) +ALL = ~EMPTY +STORAGE = Effects.STORAGE +TRANSIENT = Effects.TRANSIENT +MEMORY = Effects.MEMORY +MSIZE = Effects.MSIZE +IMMUTABLES = Effects.IMMUTABLES +RETURNDATA = Effects.RETURNDATA +LOG = Effects.LOG +BALANCE = Effects.BALANCE +EXTCODE = Effects.EXTCODE + + +_writes = { + "sstore": STORAGE, + "tstore": TRANSIENT, + "mstore": MEMORY, + "istore": IMMUTABLES, + "call": ALL ^ IMMUTABLES, + "delegatecall": ALL ^ IMMUTABLES, + "staticcall": MEMORY | RETURNDATA, + "create": ALL ^ (MEMORY | IMMUTABLES), + "create2": ALL ^ (MEMORY | IMMUTABLES), + "invoke": ALL, # could be smarter, look up the effects of the invoked function + "log": LOG, + "dloadbytes": MEMORY, + "returndatacopy": MEMORY, + "calldatacopy": MEMORY, + "codecopy": MEMORY, + "extcodecopy": MEMORY, + "mcopy": MEMORY, +} + +_reads = { + "sload": STORAGE, + "tload": TRANSIENT, + "iload": IMMUTABLES, + "mload": MEMORY, + "mcopy": MEMORY, + "call": ALL, + "delegatecall": ALL, + "staticcall": ALL, + "create": ALL, + "create2": ALL, + "invoke": ALL, + "returndatasize": RETURNDATA, + "returndatacopy": RETURNDATA, + "balance": BALANCE, + "selfbalance": BALANCE, + "extcodecopy": EXTCODE, + "selfdestruct": BALANCE, # may modify code, but after the transaction + "log": MEMORY, + "revert": MEMORY, + "return": MEMORY, + "sha3": MEMORY, + "msize": MSIZE, +} + +reads = _reads.copy() +writes = _writes.copy() + +for k, v in reads.items(): + if MEMORY in v: + if k not in writes: + writes[k] = EMPTY + writes[k] |= MSIZE + +for k, v in writes.items(): + if MEMORY in v: + writes[k] |= MSIZE From c02d2d8c5ed8c904c31b7f3f937ab01781fc9891 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Fri, 4 Oct 2024 22:20:03 -0400 Subject: [PATCH 09/13] feat[docs]: add installation via pipx and uv (#4274) add installation instructions via `pipx` and `uv tool`. add a note that `pip install vyper` is not generally recommended to do site-wide. --- docs/installing-vyper.rst | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/installing-vyper.rst b/docs/installing-vyper.rst index 515d88f87c..0c7d54903f 100644 --- a/docs/installing-vyper.rst +++ b/docs/installing-vyper.rst @@ -11,11 +11,32 @@ any errors. - To use Try Vyper, go to https://try.vyperlang.org and log in (requires Github login). - To use remix, go to https://remix.ethereum.org and activate the vyper-remix plugin in the Plugin manager. + +Standalone +********** + +The Vyper CLI can be installed with any ``pip`` compatible tool, for example, ``pipx`` or ``uv tool``. If you do not have ``pipx`` or ``uv`` installed, first, go to the respective tool's installation page: + +- https://github.com/pypa/pipx?tab=readme-ov-file +- https://github.com/astral-sh/uv?tab=readme-ov-file#uv + +Then, the command to install Vyper would be + +:: + + pipx install vyper + +Or, + +:: + + uv tool install vyper + + Binaries ******** -Vyper binaries for Windows, Mac and Linux are available for download from the `GitHub releases page -`_. +Alternatively, prebuilt Vyper binaries for Windows, Mac and Linux are available for download from the GitHub releases page: https://github.com/vyperlang/vyper/releases. PIP @@ -30,6 +51,11 @@ follow the instructions from the official `Python website Date: Mon, 7 Oct 2024 11:58:21 -0400 Subject: [PATCH 10/13] fix[tool]: update VarAccess pickle implementation (#4270) fix a bug where unpickling `annotated_vyper_module` would lead to a crash: ``` AttributeError: 'VarAccess' object has no attribute 'variable' ``` this is a blocker for tooling, for instance, titanoboa relies on pickle/unpickle to cache `CompilerData` objects: https://github.com/vyperlang/titanoboa/blob/86df8936654db2068641/boa/util/disk_cache.py#L65-L66 the underlying issue is that `pickle.loads()` calls `obj.__hash__()` for objects that are keys in a hashed data structure - namely, dicts, sets and frozensets. this causes a crash when there is a cycle in the object graph, because the object is not fully instantiated at the time that `__hash__()` is called. this is a cpython issue, reported at python/cpython#124937. @serhiy-storchaka suggested the approach taken in this PR, which breaks the loop before pickling: https://github.com/python/cpython/issues/124937#issuecomment-2392227290 note that the implementation of `__reduce__()` in this PR is safe, since there is no cycle in the hash function itself, since the recursion breaks in `VarInfo.__hash__()`. in other words, there is no possibility of `VarAccess.__hash__()` changing mid-way through reconstructing the object. --- tests/integration/test_pickle_ast.py | 19 +++++++++++++++++++ vyper/semantics/analysis/base.py | 14 ++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_pickle_ast.py diff --git a/tests/integration/test_pickle_ast.py b/tests/integration/test_pickle_ast.py new file mode 100644 index 0000000000..2c6144603a --- /dev/null +++ b/tests/integration/test_pickle_ast.py @@ -0,0 +1,19 @@ +import copy +import pickle + +from vyper.compiler.phases import CompilerData + + +def test_pickle_ast(): + code = """ +@external +def foo(): + self.bar() + y: uint256 = 5 + x: uint256 = 5 +def bar(): + pass + """ + f = CompilerData(code) + copy.deepcopy(f.annotated_vyper_module) + pickle.loads(pickle.dumps(f.annotated_vyper_module)) diff --git a/vyper/semantics/analysis/base.py b/vyper/semantics/analysis/base.py index 65bc8df3ab..982b6eb01d 100644 --- a/vyper/semantics/analysis/base.py +++ b/vyper/semantics/analysis/base.py @@ -1,5 +1,5 @@ import enum -from dataclasses import dataclass +from dataclasses import dataclass, fields from functools import cached_property from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union @@ -234,6 +234,17 @@ class VarAccess: # A sentinel indicating a subscript access SUBSCRIPT_ACCESS: ClassVar[Any] = object() + # custom __reduce__ and _produce implementations to work around + # a pickle bug. + # see https://github.com/python/cpython/issues/124937#issuecomment-2392227290 + def __reduce__(self): + dict_obj = {f.name: getattr(self, f.name) for f in fields(self)} + return self.__class__._produce, (dict_obj,) + + @classmethod + def _produce(cls, data): + return cls(**data) + @cached_property def attrs(self): ret = [] @@ -286,7 +297,6 @@ def __post_init__(self): for attr in should_match: if getattr(self.var_info, attr) != getattr(self, attr): raise CompilerPanic(f"Bad analysis: non-matching {attr}: {self}") - self._writes: OrderedSet[VarAccess] = OrderedSet() self._reads: OrderedSet[VarAccess] = OrderedSet() From d8f4032355773cd5e57e8902bd50808490de39b5 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 10 Oct 2024 13:43:41 -0400 Subject: [PATCH 11/13] feat[venom]: add small heuristic for cleaning input stack (#4251) this commit adds a small heuristic for cleaning the input stack from `cfg_in`, which is to pop the shallowest items first. it also cleans up the code a little bit and adds stronger preconditions to the code. a further optimization would be to try to align the output stack as best as possible at the `jnz`, but that belongs in the DFT pass. --- vyper/venom/venom_to_assembly.py | 41 +++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 56228d53d2..092f2f78ee 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -295,7 +295,8 @@ def _generate_evm_for_basicblock_r( asm.append(f"_sym_{basicblock.label}") asm.append("JUMPDEST") - self.clean_stack_from_cfg_in(asm, basicblock, stack) + if len(basicblock.cfg_in) == 1: + self.clean_stack_from_cfg_in(asm, basicblock, stack) all_insts = sorted(basicblock.instructions, key=lambda x: x.opcode != "param") @@ -321,26 +322,28 @@ def _generate_evm_for_basicblock_r( def clean_stack_from_cfg_in( self, asm: list, basicblock: IRBasicBlock, stack: StackModel ) -> None: - if len(basicblock.cfg_in) == 0: - return - - to_pop = OrderedSet[IRVariable]() - for in_bb in basicblock.cfg_in: - # inputs is the input variables we need from in_bb - inputs = self.liveness_analysis.input_vars_from(in_bb, basicblock) - - # layout is the output stack layout for in_bb (which works - # for all possible cfg_outs from the in_bb). - layout = in_bb.out_vars - - # pop all the stack items which in_bb produced which we don't need. - to_pop |= layout.difference(inputs) - + # the input block is a splitter block, like jnz or djmp + assert len(basicblock.cfg_in) == 1 + in_bb = basicblock.cfg_in.first() + assert len(in_bb.cfg_out) > 1 + + # inputs is the input variables we need from in_bb + inputs = self.liveness_analysis.input_vars_from(in_bb, basicblock) + + # layout is the output stack layout for in_bb (which works + # for all possible cfg_outs from the in_bb, in_bb is responsible + # for making sure its output stack layout works no matter which + # bb it jumps into). + layout = in_bb.out_vars + to_pop = list(layout.difference(inputs)) + + # small heuristic: pop from shallowest first. + to_pop.sort(key=lambda var: -stack.get_depth(var)) + + # NOTE: we could get more fancy and try to optimize the swap + # operations here, there is probably some more room for optimization. for var in to_pop: depth = stack.get_depth(var) - # don't pop phantom phi inputs - if depth is StackModel.NOT_IN_STACK: - continue if depth != 0: self.swap(asm, stack, depth) From fdc05d6029816ef57294e98753d9a3d60c761ab8 Mon Sep 17 00:00:00 2001 From: Harry Kalogirou Date: Thu, 10 Oct 2024 22:27:08 +0300 Subject: [PATCH 12/13] fix[venom]: promote additional memory locations to variables (#4039) promote additional memory locations (specifically, internal function params allocated with `palloca`) to stack variables. --------- Co-authored-by: Charles Cooper --- vyper/venom/ir_node_to_venom.py | 2 +- vyper/venom/passes/mem2var.py | 47 ++++++++++++++++++++++++++------ vyper/venom/passes/sccp/sccp.py | 2 +- vyper/venom/venom_to_assembly.py | 4 +-- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/vyper/venom/ir_node_to_venom.py b/vyper/venom/ir_node_to_venom.py index 85172c70e1..e30f27f480 100644 --- a/vyper/venom/ir_node_to_venom.py +++ b/vyper/venom/ir_node_to_venom.py @@ -548,7 +548,7 @@ def emit_body_blocks(): _global_symbols[ir.value] = ptr elif ir.value.startswith("$palloca") and ir.value not in _global_symbols: alloca = ir.passthrough_metadata["alloca"] - ptr = fn.get_basic_block().append_instruction("store", alloca.offset) + ptr = fn.get_basic_block().append_instruction("palloca", alloca.offset, alloca.size) _global_symbols[ir.value] = ptr return _global_symbols.get(ir.value) or symbols.get(ir.value) diff --git a/vyper/venom/passes/mem2var.py b/vyper/venom/passes/mem2var.py index f4a37f5abb..74977d6279 100644 --- a/vyper/venom/passes/mem2var.py +++ b/vyper/venom/passes/mem2var.py @@ -1,8 +1,7 @@ -from vyper.utils import OrderedSet from vyper.venom.analysis.cfg import CFGAnalysis from vyper.venom.analysis.dfg import DFGAnalysis from vyper.venom.analysis.liveness import LivenessAnalysis -from vyper.venom.basicblock import IRBasicBlock, IRInstruction, IRVariable +from vyper.venom.basicblock import IRInstruction, IRVariable from vyper.venom.function import IRFunction from vyper.venom.passes.base_pass import IRPass @@ -14,7 +13,6 @@ class Mem2Var(IRPass): """ function: IRFunction - defs: dict[IRVariable, OrderedSet[IRBasicBlock]] def run_pass(self): self.analyses_cache.request_analysis(CFGAnalysis) @@ -22,13 +20,20 @@ def run_pass(self): self.var_name_count = 0 for var, inst in dfg.outputs.items(): - if inst.opcode != "alloca": - continue - self._process_alloca_var(dfg, var) + if inst.opcode == "alloca": + self._process_alloca_var(dfg, var) + elif inst.opcode == "palloca": + self._process_palloca_var(dfg, inst, var) self.analyses_cache.invalidate_analysis(DFGAnalysis) self.analyses_cache.invalidate_analysis(LivenessAnalysis) + def _mk_varname(self, varname: str): + varname = varname.removeprefix("%") + varname = f"var{varname}_{self.var_name_count}" + self.var_name_count += 1 + return varname + def _process_alloca_var(self, dfg: DFGAnalysis, var: IRVariable): """ Process alloca allocated variable. If it is only used by mstore/mload/return @@ -40,8 +45,7 @@ def _process_alloca_var(self, dfg: DFGAnalysis, var: IRVariable): elif all([inst.opcode == "mstore" for inst in uses]): return elif all([inst.opcode in ["mstore", "mload", "return"] for inst in uses]): - var_name = f"addr{var.name}_{self.var_name_count}" - self.var_name_count += 1 + var_name = self._mk_varname(var.name) for inst in uses: if inst.opcode == "mstore": inst.opcode = "store" @@ -52,7 +56,32 @@ def _process_alloca_var(self, dfg: DFGAnalysis, var: IRVariable): inst.operands = [IRVariable(var_name)] elif inst.opcode == "return": bb = inst.parent - idx = bb.instructions.index(inst) + idx = len(bb.instructions) - 1 + assert inst == bb.instructions[idx] # sanity bb.insert_instruction( IRInstruction("mstore", [IRVariable(var_name), inst.operands[1]]), idx ) + + def _process_palloca_var(self, dfg: DFGAnalysis, palloca_inst: IRInstruction, var: IRVariable): + """ + Process alloca allocated variable. If it is only used by mstore/mload + instructions, it is promoted to a stack variable. Otherwise, it is left as is. + """ + uses = dfg.get_uses(var) + if not all(inst.opcode in ["mstore", "mload"] for inst in uses): + return + + var_name = self._mk_varname(var.name) + + palloca_inst.opcode = "mload" + palloca_inst.operands = [palloca_inst.operands[0]] + palloca_inst.output = IRVariable(var_name) + + for inst in uses: + if inst.opcode == "mstore": + inst.opcode = "store" + inst.output = IRVariable(var_name) + inst.operands = [inst.operands[0]] + elif inst.opcode == "mload": + inst.opcode = "store" + inst.operands = [IRVariable(var_name)] diff --git a/vyper/venom/passes/sccp/sccp.py b/vyper/venom/passes/sccp/sccp.py index 013583ec63..8596bc8405 100644 --- a/vyper/venom/passes/sccp/sccp.py +++ b/vyper/venom/passes/sccp/sccp.py @@ -179,7 +179,7 @@ def _visit_phi(self, inst: IRInstruction): def _visit_expr(self, inst: IRInstruction): opcode = inst.opcode - if opcode in ["store", "alloca"]: + if opcode in ["store", "alloca", "palloca"]: assert inst.output is not None, "Got store/alloca without output" out = self._eval_from_lattice(inst.operands[0]) self._set_lattice(inst.output, out) diff --git a/vyper/venom/venom_to_assembly.py b/vyper/venom/venom_to_assembly.py index 092f2f78ee..21e3b5e5aa 100644 --- a/vyper/venom/venom_to_assembly.py +++ b/vyper/venom/venom_to_assembly.py @@ -363,7 +363,7 @@ def _generate_evm_for_instruction( if opcode in ["jmp", "djmp", "jnz", "invoke"]: operands = list(inst.get_non_label_operands()) - elif opcode == "alloca": + elif opcode in ("alloca", "palloca"): offset, _size = inst.operands operands = [offset] @@ -463,7 +463,7 @@ def _generate_evm_for_instruction( # Step 5: Emit the EVM instruction(s) if opcode in _ONE_TO_ONE_INSTRUCTIONS: assembly.append(opcode.upper()) - elif opcode == "alloca": + elif opcode in ("alloca", "palloca"): pass elif opcode == "param": pass From 4845fd4826683167eca991be4e4da5316f6993bb Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Thu, 10 Oct 2024 19:27:33 -0400 Subject: [PATCH 13/13] feat[tool]: add all imported modules to `-f annotated_ast` output (#4209) this commit adds all imported modules to `-f annotated_ast` output. this includes recursively imported modules. this is a tooling feature, so that tools don't need to invoke the compiler multiple and inspect import statements in order to find the ASTs for imported modules. --------- Co-authored-by: cyberthirst --- tests/unit/ast/test_ast_dict.py | 58 ++++++++++++++++++++++++++++++--- vyper/ast/nodes.py | 6 ++++ vyper/ast/nodes.pyi | 1 + vyper/compiler/output.py | 28 +++++++++++++++- 4 files changed, 88 insertions(+), 5 deletions(-) diff --git a/tests/unit/ast/test_ast_dict.py b/tests/unit/ast/test_ast_dict.py index 07da3c0ace..c9d7248809 100644 --- a/tests/unit/ast/test_ast_dict.py +++ b/tests/unit/ast/test_ast_dict.py @@ -1,3 +1,4 @@ +import copy import json from vyper import compiler @@ -216,24 +217,27 @@ def foo(): input_bundle = make_input_bundle({"lib1.vy": lib1, "main.vy": main}) lib1_file = input_bundle.load_file("lib1.vy") - out = compiler.compile_from_file_input( + lib1_out = compiler.compile_from_file_input( lib1_file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] ) - lib1_ast = out["annotated_ast_dict"]["ast"] + + lib1_ast = copy.deepcopy(lib1_out["annotated_ast_dict"]["ast"]) lib1_sha256sum = lib1_ast.pop("source_sha256sum") assert lib1_sha256sum == lib1_file.sha256sum to_strip = NODE_SRC_ATTRIBUTES + ("resolved_path", "variable_reads", "variable_writes") _strip_source_annotations(lib1_ast, to_strip=to_strip) main_file = input_bundle.load_file("main.vy") - out = compiler.compile_from_file_input( + main_out = compiler.compile_from_file_input( main_file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] ) - main_ast = out["annotated_ast_dict"]["ast"] + main_ast = main_out["annotated_ast_dict"]["ast"] main_sha256sum = main_ast.pop("source_sha256sum") assert main_sha256sum == main_file.sha256sum _strip_source_annotations(main_ast, to_strip=to_strip) + assert main_out["annotated_ast_dict"]["imports"][0] == lib1_out["annotated_ast_dict"]["ast"] + # TODO: would be nice to refactor this into bunch of small test cases assert main_ast == { "ast_type": "Module", @@ -1776,3 +1780,49 @@ def qux2(): }, } ] + + +def test_annotated_ast_export_recursion(make_input_bundle): + sources = { + "main.vy": """ +import lib1 + +@external +def foo(): + lib1.foo() + """, + "lib1.vy": """ +import lib2 + +def foo(): + lib2.foo() + """, + "lib2.vy": """ +def foo(): + pass + """, + } + + input_bundle = make_input_bundle(sources) + + def compile_and_get_ast(file_name): + file = input_bundle.load_file(file_name) + output = compiler.compile_from_file_input( + file, input_bundle=input_bundle, output_formats=["annotated_ast_dict"] + ) + return output["annotated_ast_dict"] + + lib1_ast = compile_and_get_ast("lib1.vy")["ast"] + lib2_ast = compile_and_get_ast("lib2.vy")["ast"] + main_out = compile_and_get_ast("main.vy") + + lib1_import_ast = main_out["imports"][1] + lib2_import_ast = main_out["imports"][0] + + # path is once virtual, once libX.vy + # type contains name which is based on path + keys = [s for s in lib1_import_ast.keys() if s not in {"path", "type"}] + + for key in keys: + assert lib1_ast[key] == lib1_import_ast[key] + assert lib2_ast[key] == lib2_import_ast[key] diff --git a/vyper/ast/nodes.py b/vyper/ast/nodes.py index 991edeca6e..d3c721dbfb 100644 --- a/vyper/ast/nodes.py +++ b/vyper/ast/nodes.py @@ -949,6 +949,12 @@ def validate(self): class Ellipsis(Constant): __slots__ = () + def to_dict(self): + ast_dict = super().to_dict() + # python ast ellipsis() is not json serializable; use a string + ast_dict["value"] = self.node_source_code + return ast_dict + class Dict(ExprNode): __slots__ = ("keys", "values") diff --git a/vyper/ast/nodes.pyi b/vyper/ast/nodes.pyi index 58c7d0b2e4..3e3a9a62b2 100644 --- a/vyper/ast/nodes.pyi +++ b/vyper/ast/nodes.pyi @@ -70,6 +70,7 @@ class TopLevel(VyperNode): class Module(TopLevel): path: str = ... resolved_path: str = ... + source_id: int = ... def namespace(self) -> Any: ... # context manager class FunctionDef(TopLevel): diff --git a/vyper/compiler/output.py b/vyper/compiler/output.py index 09d299b90d..d04b677b3e 100644 --- a/vyper/compiler/output.py +++ b/vyper/compiler/output.py @@ -3,7 +3,8 @@ from collections import deque from pathlib import PurePath -from vyper.ast import ast_to_dict +import vyper.ast as vy_ast +from vyper.ast.utils import ast_to_dict from vyper.codegen.ir_node import IRnode from vyper.compiler.output_bundle import SolcJSONWriter, VyperArchiveWriter from vyper.compiler.phases import CompilerData @@ -11,7 +12,9 @@ from vyper.evm import opcodes from vyper.exceptions import VyperException from vyper.ir import compile_ir +from vyper.semantics.analysis.base import ModuleInfo from vyper.semantics.types.function import FunctionVisibility, StateMutability +from vyper.semantics.types.module import InterfaceT from vyper.typing import StorageLayout from vyper.utils import vyper_warn from vyper.warnings import ContractSizeLimitWarning @@ -26,9 +29,32 @@ def build_ast_dict(compiler_data: CompilerData) -> dict: def build_annotated_ast_dict(compiler_data: CompilerData) -> dict: + module_t = compiler_data.annotated_vyper_module._metadata["type"] + # get all reachable imports including recursion + imported_module_infos = module_t.reachable_imports + unique_modules: dict[str, vy_ast.Module] = {} + for info in imported_module_infos: + if isinstance(info.typ, InterfaceT): + ast = info.typ.decl_node + if ast is None: # json abi + continue + else: + assert isinstance(info.typ, ModuleInfo) + ast = info.typ.module_t._module + + assert isinstance(ast, vy_ast.Module) # help mypy + # use resolved_path for uniqueness, since Module objects can actually + # come from multiple InputBundles (particularly builtin interfaces), + # so source_id is not guaranteed to be unique. + if ast.resolved_path in unique_modules: + # sanity check -- objects must be identical + assert unique_modules[ast.resolved_path] is ast + unique_modules[ast.resolved_path] = ast + annotated_ast_dict = { "contract_name": str(compiler_data.contract_path), "ast": ast_to_dict(compiler_data.annotated_vyper_module), + "imports": [ast_to_dict(ast) for ast in unique_modules.values()], } return annotated_ast_dict