From 8f62eac8cf2630dc96f2d72ea63acdc9ab67df6f Mon Sep 17 00:00:00 2001 From: SHAKOTN Date: Wed, 12 Jul 2023 12:02:57 +0300 Subject: [PATCH 1/8] feat: cleanups in addresses module. Also: - Add munch library - Add new deployments property --- bal_addresses/addresses.py | 105 +++++++++++++++++---------------- bal_addresses/exceptions.py | 6 ++ bal_addresses/requirements.txt | 3 +- 3 files changed, 61 insertions(+), 53 deletions(-) create mode 100644 bal_addresses/exceptions.py diff --git a/bal_addresses/addresses.py b/bal_addresses/addresses.py index 68344810..726e781c 100644 --- a/bal_addresses/addresses.py +++ b/bal_addresses/addresses.py @@ -1,68 +1,72 @@ - import json -from web3 import Web3 + import requests from dotmap import DotMap +from web3 import Web3 -## Expose some config for bootstrapping, maybe there is a better way to do this without so many github hits but allowing this to be used before invoking the class. -chains = requests.get(f"https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/extras/chains.json").json() -CHAIN_IDS_BY_NAME = chains["CHAIN_IDS_BY_NAME"] -SCANNERS_BY_CHAIN = chains["SCANNERS_BY_CHAIN"] - - +from bal_addresses.exceptions import MultipleMatchesError +from bal_addresses.exceptions import NoResultError + +GITHUB_MONOREPO_RAW = ( + "https://raw.githubusercontent.com/balancer-labs/balancer-v2-monorepo/master" +) +GITHUB_MONOREPO_NICE = ( + "https://github.com/balancer/balancer-v2-monorepo/blob/master" +) +GITHUB_DEPLOYMENTS_RAW = ( + "https://raw.githubusercontent.com/balancer/balancer-deployments/master" +) +GITHUB_DEPLOYMENTS_NICE = "https://github.com/balancer/balancer-deployments/blob/master" +GITHUB_RAW_OUTPUTS = ( + "https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/outputs" +) +ZERO_ADDRESS = "0x0000000000000000000000000000000000000000" -### Main class class AddrBook: - chains = DotMap(requests.get( - f"https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/extras/chains.json").json()) - GITHUB_MONOREPO_RAW = "https://raw.githubusercontent.com/balancer-labs/balancer-v2-monorepo/master" - GITHUB_MONOREPO_NICE = "https://github.com/balancer/balancer-v2-monorepo/blob/master" - GITHUB_DEPLOYMENTS_RAW = "https://raw.githubusercontent.com/balancer/balancer-deployments/master" - GITHUB_DEPLOYMENTS_NICE = "https://github.com/balancer/balancer-deployments/blob/master" - GITHUB_RAW_OUTPUTS = "https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/outputs" - ZERO_ADDRESS = "0x0000000000000000000000000000000000000000" - CHAIN_IDS_BY_NAME = chains["CHAIN_IDS_BY_NAME"] - SCANNERS_BY_CHAIN = chains["SCANNERS_BY_CHAIN"] fullbook = requests.get(f"{GITHUB_RAW_OUTPUTS}/addressbook.json").json() - fx_description_by_name = requests.get("https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/extras/func_desc_by_name.json").json - - ### Errors - class MultipleMatchesError(Exception): - pass - - class NoResultError(Exception): - pass + chains = requests.get( + "https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/extras/chains.json" + ).json() def __init__(self, chain, jsonfile=False): - self.jsonfile=jsonfile + self.jsonfile = jsonfile self.chain = chain self.dotmap = self.build_dotmap() - deployments = requests.get(f"{self.GITHUB_RAW_OUTPUTS}/deployments.json").json() + deployments = requests.get(f"{GITHUB_RAW_OUTPUTS}/deployments.json").json() try: - dold = deployments["old"][chain] - except: + dold = deployments["old"][chain] + except Exception: dold = {} try: dactive = deployments["active"][chain] - except: + except Exception: dactive = {} self.deployments_only = DotMap(dactive | dold) try: - self.flatbook = requests.get(f"{self.GITHUB_RAW_OUTPUTS}/{chain}.json").json() - self.reversebook = DotMap(requests.get(f"{self.GITHUB_RAW_OUTPUTS}/{chain}_reverse.json").json()) - except: - self.flatbook = {"zero/zero": self.ZERO_ADDRESS } - self.reversebook = {self.ZERO_ADDRESS: "zero/zero"} - + self.flatbook = requests.get(f"{GITHUB_RAW_OUTPUTS}/{chain}.json").json() + self.reversebook = DotMap( + requests.get(f"{GITHUB_RAW_OUTPUTS}/{chain}_reverse.json").json()) + except Exception: + self.flatbook = {"zero/zero": ZERO_ADDRESS} + self.reversebook = {ZERO_ADDRESS: "zero/zero"} + + self._deployments = None + + @property + def deployments(self): + if self._deployments is not None: + return self._deployments + for chain_name in self.chains['CHAIN_IDS_BY_NAME'].keys(): + print(chain_name) def search_unique(self, substr): results = [s for s in self.flatbook.keys() if substr in s] if len(results) > 1: - raise self.MultipleMatchesError(f"{substr} Multiple matches found: {results}") - if len(results) < 1: - raise self.NoResultError(f"{substr}") + raise MultipleMatchesError(f"{substr} Multiple matches found: {results}") + if len(results) < 1: + raise NoResultError(f"{substr}") return results[0] def search_many(self, substr): @@ -76,12 +80,11 @@ def latest_contract(self, contract_name): if contract_name in contractData.keys(): deployments.append(deployment) if len(deployments) == 0: - raise self.NoResultError(contract_name) + raise NoResultError(contract_name) deployments.sort(reverse=True) return self.deployments_only[deployments[0]][contract_name] - - def checksum_address_dict(addresses): + def checksum_address_dict(self, addresses): """ convert addresses to their checksum variant taken from a (nested) dict """ @@ -101,8 +104,9 @@ def build_dotmap(self): fullbook = json.load(f) else: fullbook = self.fullbook - return(DotMap(fullbook["active"].get(self.chain, {}) | fullbook["old"].get(self.chain, {}))) - ### Checksum one more time for good measure + return ( + DotMap(fullbook["active"].get(self.chain, {}) | fullbook["old"].get(self.chain, {}))) + # Checksum one more time for good measure def flatten_dict(self, d, parent_key='', sep='/'): items = [] @@ -116,14 +120,11 @@ def flatten_dict(self, d, parent_key='', sep='/'): def generate_flatbook(self): print(f"Generating Addressbook for {self.chain}") - monorepo_addresses = {} - dupContracts = {} ab = dict(self.dotmap) - return(self.flatten_dict(ab)) - + return self.flatten_dict(ab) -# Version outside class to allow for recursion on the uninitialized class +# Version outside class to allow for recursion on the uninitialized class def checksum_address_dict(addresses): """ convert addresses to their checksum variant taken from a (nested) dict @@ -136,4 +137,4 @@ def checksum_address_dict(addresses): checksummed[k] = checksum_address_dict(v) else: print(k, v, "formatted incorrectly") - return checksummed \ No newline at end of file + return checksummed diff --git a/bal_addresses/exceptions.py b/bal_addresses/exceptions.py new file mode 100644 index 00000000..d3c180e0 --- /dev/null +++ b/bal_addresses/exceptions.py @@ -0,0 +1,6 @@ +class MultipleMatchesError(Exception): + pass + + +class NoResultError(Exception): + pass diff --git a/bal_addresses/requirements.txt b/bal_addresses/requirements.txt index 55df9cf9..05cf9b4a 100644 --- a/bal_addresses/requirements.txt +++ b/bal_addresses/requirements.txt @@ -2,4 +2,5 @@ pathlib>=1.0 requests pandas web3==5.31.3 -dotmap \ No newline at end of file +dotmap +munch==4.0.0 From 0fcb475919e77389d32401f429a615ec4ebd71bd Mon Sep 17 00:00:00 2001 From: SHAKOTN Date: Wed, 12 Jul 2023 12:38:02 +0300 Subject: [PATCH 2/8] feat: implement deployment fetching --- bal_addresses/addresses.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/bal_addresses/addresses.py b/bal_addresses/addresses.py index 726e781c..07e2243c 100644 --- a/bal_addresses/addresses.py +++ b/bal_addresses/addresses.py @@ -1,7 +1,9 @@ import json +from typing import Dict import requests from dotmap import DotMap +from munch import Munch from web3 import Web3 from bal_addresses.exceptions import MultipleMatchesError @@ -55,11 +57,33 @@ def __init__(self, chain, jsonfile=False): self._deployments = None @property - def deployments(self): + def deployments(self) -> Munch: + """ + Get the deployments for all chains in a form of a Munch object + """ if self._deployments is not None: return self._deployments + self._deployments = Munch() for chain_name in self.chains['CHAIN_IDS_BY_NAME'].keys(): - print(chain_name) + chain_deployments = requests.get( + f"{GITHUB_DEPLOYMENTS_RAW}/addresses/{chain_name}.json" + ) + if chain_deployments.ok: + # Remove date from key + processed_deployment = self._process_deployment(chain_deployments.json()) + setattr(self._deployments, chain_name, Munch.fromDict(processed_deployment)) + return self._deployments + + def _process_deployment(self, deployment: Dict) -> Dict: + """ + Process deployment to remove date from key and replace - with _ + """ + processed_deployment = {} + for k, v in deployment.items(): + # lstrip date in format YYYYMMDD-: + # Change all - to underscores + processed_deployment[k.lstrip("0123456789-").replace("-", "_")] = v + return processed_deployment def search_unique(self, substr): results = [s for s in self.flatbook.keys() if substr in s] From db6b8a7223b6fdca33c15c1ebd380d664883299b Mon Sep 17 00:00:00 2001 From: SHAKOTN Date: Wed, 12 Jul 2023 12:46:16 +0300 Subject: [PATCH 3/8] tests: add tests package and CI pipeline --- .github/workflows/test.yml | 26 ++++++++++++++++++++++++++ Makefile | 6 ++++++ bal_addresses/requirements-dev.txt | 4 ++++ tests/__init__.py | 0 tests/test_addresses.py | 5 +++++ 5 files changed, 41 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100644 Makefile create mode 100644 bal_addresses/requirements-dev.txt create mode 100644 tests/__init__.py create mode 100644 tests/test_addresses.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..7955a279 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,26 @@ +name: Tests + +on: [push, pull_request] + +jobs: + build: + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: [3.9, '3.10'] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + make + - name: Run tests + run: | + make ci diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..7254216d --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +.PHONY: docs +init: + pip install -e .[socks] + pip install -r bal_addresses/requirements-dev.txt +ci: + pytest diff --git a/bal_addresses/requirements-dev.txt b/bal_addresses/requirements-dev.txt new file mode 100644 index 00000000..c3b82fe9 --- /dev/null +++ b/bal_addresses/requirements-dev.txt @@ -0,0 +1,4 @@ +pytest-mock +responses +pytest-cov +pytest==7.4.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_addresses.py b/tests/test_addresses.py new file mode 100644 index 00000000..134407e8 --- /dev/null +++ b/tests/test_addresses.py @@ -0,0 +1,5 @@ +from bal_addresses import AddrBook + + +def test_addresses(): + a = AddrBook("mainnet") From a1e3f263f157c7ff2b848f03cc238c46a7fd6f7f Mon Sep 17 00:00:00 2001 From: SHAKOTN Date: Wed, 12 Jul 2023 13:11:06 +0300 Subject: [PATCH 4/8] chore: add some basic tests --- bal_addresses/addresses.py | 25 ++++++++------ tests/test_addresses.py | 69 +++++++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/bal_addresses/addresses.py b/bal_addresses/addresses.py index 07e2243c..6f0051a2 100644 --- a/bal_addresses/addresses.py +++ b/bal_addresses/addresses.py @@ -1,5 +1,6 @@ import json from typing import Dict +from typing import Optional import requests from dotmap import DotMap @@ -57,23 +58,27 @@ def __init__(self, chain, jsonfile=False): self._deployments = None @property - def deployments(self) -> Munch: + def deployments(self) -> Optional[Munch]: """ Get the deployments for all chains in a form of a Munch object """ if self._deployments is not None: return self._deployments - self._deployments = Munch() - for chain_name in self.chains['CHAIN_IDS_BY_NAME'].keys(): - chain_deployments = requests.get( - f"{GITHUB_DEPLOYMENTS_RAW}/addresses/{chain_name}.json" - ) - if chain_deployments.ok: - # Remove date from key - processed_deployment = self._process_deployment(chain_deployments.json()) - setattr(self._deployments, chain_name, Munch.fromDict(processed_deployment)) + else: + self.populate_deployments() + return self._deployments + def populate_deployments(self) -> None: + chain_deployments = requests.get( + f"{GITHUB_DEPLOYMENTS_RAW}/addresses/{self.chain}.json" + ) + if chain_deployments.ok: + self._deployments = Munch() + # Remove date from key + processed_deployment = self._process_deployment(chain_deployments.json()) + self._deployments = Munch.fromDict(processed_deployment) + def _process_deployment(self, deployment: Dict) -> Dict: """ Process deployment to remove date from key and replace - with _ diff --git a/tests/test_addresses.py b/tests/test_addresses.py index 134407e8..b16dde31 100644 --- a/tests/test_addresses.py +++ b/tests/test_addresses.py @@ -1,5 +1,72 @@ +import pytest +import responses + from bal_addresses import AddrBook -def test_addresses(): +@responses.activate +def test_deployments_populated(): + responses.add( + responses.GET, + "https://raw.githubusercontent.com/BalancerMaxis" + "/bal_addresses/main/outputs/deployments.json", + json={ + "BFactory": "0x9424B1412450D0f8Fc2255FAf6046b98213B76Bd", + } + ) + responses.add( + responses.GET, + "https://raw.githubusercontent.com/balancer" + "/balancer-deployments/master/addresses/mainnet.json", + json={ + "20210418-vault": { + "contracts": [ + { + "name": "Vault", + "address": "0xBA12222222228d8Ba445958a75a0704d566BF2C8" + }, + { + "name": "BalancerHelpers", + "address": "0x5aDDCCa35b7A0D07C74063c48700C8590E87864E" + }, + { + "name": "ProtocolFeesCollector", + "address": "0xce88686553686DA562CE7Cea497CE749DA109f9F" + } + ], + "status": "ACTIVE" + } + } + ) + a = AddrBook("mainnet") + + a.populate_deployments() + assert a.deployments.vault.status == "ACTIVE" + assert a.deployments.vault.contracts[0].name == "Vault" + assert a.deployments.vault.contracts[1].name == "BalancerHelpers" + # Make sure that when we try to access a non-existing attribute, we get an error + with pytest.raises(AttributeError): + assert a.deployments.vault.non_existing_attribute + + +@responses.activate +def test_deployments_not_populated(): + responses.add( + responses.GET, + "https://raw.githubusercontent.com/BalancerMaxis" + "/bal_addresses/main/outputs/deployments.json", + json={ + "BFactory": "0x9424B1412450D0f8Fc2255FAf6046b98213B76Bd", + } + ) + responses.add( + responses.GET, + "https://raw.githubusercontent.com/balancer" + "/balancer-deployments/master/addresses/mainnet.json", + json={}, + status=404 + ) a = AddrBook("mainnet") + assert a.deployments is None + with pytest.raises(AttributeError): + assert a.deployments.vault.non_existing_attribute From ba3df912be170ef1284dd889359c75bec7a250c9 Mon Sep 17 00:00:00 2001 From: SHAKOTN Date: Wed, 12 Jul 2023 14:47:05 +0300 Subject: [PATCH 5/8] chore: install common requirements --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 7254216d..4dd849d0 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ .PHONY: docs init: pip install -e .[socks] + pip install -r bal_addresses/requirements.txt pip install -r bal_addresses/requirements-dev.txt ci: pytest From 5c5a4bb3748677bf7b86ee11dacbc2ec5a7623fb Mon Sep 17 00:00:00 2001 From: SHAKOTN Date: Wed, 12 Jul 2023 15:16:04 +0300 Subject: [PATCH 6/8] tests: adding more tests --- bal_addresses/addresses.py | 6 +++++- tests/test_addresses.py | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/bal_addresses/addresses.py b/bal_addresses/addresses.py index 6f0051a2..d722fcad 100644 --- a/bal_addresses/addresses.py +++ b/bal_addresses/addresses.py @@ -87,7 +87,11 @@ def _process_deployment(self, deployment: Dict) -> Dict: for k, v in deployment.items(): # lstrip date in format YYYYMMDD-: # Change all - to underscores - processed_deployment[k.lstrip("0123456789-").replace("-", "_")] = v + deployment_identifier = k.lstrip("0123456789-").replace("-", "_") + # Flatten contracts list to dict with name as key + if isinstance(v.get('contracts'), list): + v['contracts'] = {contract['name']: contract for contract in v['contracts']} + processed_deployment[deployment_identifier] = v return processed_deployment def search_unique(self, substr): diff --git a/tests/test_addresses.py b/tests/test_addresses.py index b16dde31..a97d6a97 100644 --- a/tests/test_addresses.py +++ b/tests/test_addresses.py @@ -42,13 +42,46 @@ def test_deployments_populated(): a.populate_deployments() assert a.deployments.vault.status == "ACTIVE" - assert a.deployments.vault.contracts[0].name == "Vault" - assert a.deployments.vault.contracts[1].name == "BalancerHelpers" + assert a.deployments.vault.contracts.Vault.name == "Vault" + assert ( + a.deployments.vault.contracts.Vault.address == "0xBA12222222228d8Ba445958a75a0704d566BF2C8" + ) + assert a.deployments.vault.contracts.BalancerHelpers.name == "BalancerHelpers" # Make sure that when we try to access a non-existing attribute, we get an error with pytest.raises(AttributeError): assert a.deployments.vault.non_existing_attribute +@responses.activate +def test_deployments_invalid_format(): + """ + Make sure that library is data agnostic and can handle different formats + """ + responses.add( + responses.GET, + "https://raw.githubusercontent.com/BalancerMaxis" + "/bal_addresses/main/outputs/deployments.json", + json={ + "BFactory": "0x9424B1412450D0f8Fc2255FAf6046b98213B76Bd", + } + ) + responses.add( + responses.GET, + "https://raw.githubusercontent.com/balancer" + "/balancer-deployments/master/addresses/mainnet.json", + json={ + "20210418-vault": { + "contracts": {'name': 'Vault'}, + "status": "ACTIVE" + } + } + ) + a = AddrBook("mainnet") + + a.populate_deployments() + assert a.deployments.vault.contracts.name == "Vault" + + @responses.activate def test_deployments_not_populated(): responses.add( From b8d43ecdaa03ab45ebdffcd27f89d99a6f608570 Mon Sep 17 00:00:00 2001 From: SHAKOTN Date: Wed, 12 Jul 2023 15:22:43 +0300 Subject: [PATCH 7/8] docs: update documentation about deployments --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 3179e94e..14e52f04 100644 --- a/README.md +++ b/README.md @@ -84,3 +84,23 @@ There is also search and lookup commands ``` Most of the other functions are used by a github action which regenerates files read in by those 2 functions on a weekly basis. You can explore them if you would like. +## Using deployments: +`.deployments` attribute is an object that is lazy loaded on first access. +It has first class support in IDEs, so you can use it as a normal object. + +To use deployments information you can do the following: +```python +from bal_addresses.addresses import AddrBook + +a = AddrBook("mainnet") +# At the stage when you try to access the deployments, the data will be loaded: +a.deployments +``` + +Now you can extract information: +``` +>>> a.deployments.vault.contracts.Vault.address +'0xBA12222222228d8Ba445958a75a0704d566BF2C8' +>>> a.deployments.vault.contracts.Vault.name +'Vault' +``` From 056bc514a4a2203bd6ac4b4ed2efadde9a7c9e12 Mon Sep 17 00:00:00 2001 From: SHAKOTN Date: Thu, 13 Jul 2023 16:17:24 +0300 Subject: [PATCH 8/8] fix: move back exceptions definition as they were causing import errors --- bal_addresses/addresses.py | 11 ++++++++--- bal_addresses/exceptions.py | 6 ------ 2 files changed, 8 insertions(+), 9 deletions(-) delete mode 100644 bal_addresses/exceptions.py diff --git a/bal_addresses/addresses.py b/bal_addresses/addresses.py index d722fcad..1bbffbd3 100644 --- a/bal_addresses/addresses.py +++ b/bal_addresses/addresses.py @@ -7,9 +7,6 @@ from munch import Munch from web3 import Web3 -from bal_addresses.exceptions import MultipleMatchesError -from bal_addresses.exceptions import NoResultError - GITHUB_MONOREPO_RAW = ( "https://raw.githubusercontent.com/balancer-labs/balancer-v2-monorepo/master" ) @@ -26,6 +23,14 @@ ZERO_ADDRESS = "0x0000000000000000000000000000000000000000" +class MultipleMatchesError(Exception): + pass + + +class NoResultError(Exception): + pass + + class AddrBook: fullbook = requests.get(f"{GITHUB_RAW_OUTPUTS}/addressbook.json").json() diff --git a/bal_addresses/exceptions.py b/bal_addresses/exceptions.py deleted file mode 100644 index d3c180e0..00000000 --- a/bal_addresses/exceptions.py +++ /dev/null @@ -1,6 +0,0 @@ -class MultipleMatchesError(Exception): - pass - - -class NoResultError(Exception): - pass