Merge pull request #62 from BalancerMaxis/deployments

Deployments fetch

.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: Tests
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.9, '3.10']
+        os: [ubuntu-latest]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        make
+    - name: Run tests
+      run: |
+        make ci

Makefile

@@ -0,0 +1,7 @@
+.PHONY: docs
+init:
+	pip install -e .[socks]
+	pip install -r bal_addresses/requirements.txt
+	pip install -r bal_addresses/requirements-dev.txt
+ci:
+	pytest

README.md

@@ -84,3 +84,23 @@ There is also search and lookup commands
 ```
 Most of the other functions are used by a github action which regenerates files read in by those 2 functions on a weekly basis.  You can explore them if you would like.  
 
+## Using deployments:
+`.deployments` attribute is an object that is lazy loaded on first access.
+It has first class support in IDEs, so you can use it as a normal object.
+
+To use deployments information you can do the following:
+```python
+from bal_addresses.addresses import AddrBook
+
+a = AddrBook("mainnet")
+# At the stage when you try to access the deployments, the data will be loaded:
+a.deployments
+```
+
+Now you can extract information:
+```
+>>> a.deployments.vault.contracts.Vault.address
+'0xBA12222222228d8Ba445958a75a0704d566BF2C8'
+>>> a.deployments.vault.contracts.Vault.name
+'Vault'
+```

bal_addresses/addresses.py

@@ -1,68 +1,105 @@
-
 import json
-from web3 import Web3
+from typing import Dict
+from typing import Optional
+
 import requests
 from dotmap import DotMap
+from munch import Munch
+from web3 import Web3
 
-## Expose some config for bootstrapping, maybe there is a better way to do this without so many github hits but allowing this to be used before invoking the class.
-chains = requests.get(f"https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/extras/chains.json").json()
-CHAIN_IDS_BY_NAME = chains["CHAIN_IDS_BY_NAME"]
-SCANNERS_BY_CHAIN = chains["SCANNERS_BY_CHAIN"]
-
-
+from bal_addresses.exceptions import MultipleMatchesError
+from bal_addresses.exceptions import NoResultError
+
+GITHUB_MONOREPO_RAW = (
+    "https://raw.githubusercontent.com/balancer-labs/balancer-v2-monorepo/master"
+)
+GITHUB_MONOREPO_NICE = (
+    "https://github.com/balancer/balancer-v2-monorepo/blob/master"
+)
+GITHUB_DEPLOYMENTS_RAW = (
+    "https://raw.githubusercontent.com/balancer/balancer-deployments/master"
+)
+GITHUB_DEPLOYMENTS_NICE = "https://github.com/balancer/balancer-deployments/blob/master"
+GITHUB_RAW_OUTPUTS = (
+    "https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/outputs"
+)
+ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"
 
 
-### Main class
 class AddrBook:
-    chains = DotMap(requests.get(
-        f"https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/extras/chains.json").json())
-    GITHUB_MONOREPO_RAW = "https://raw.githubusercontent.com/balancer-labs/balancer-v2-monorepo/master"
-    GITHUB_MONOREPO_NICE = "https://github.com/balancer/balancer-v2-monorepo/blob/master"
-    GITHUB_DEPLOYMENTS_RAW = "https://raw.githubusercontent.com/balancer/balancer-deployments/master"
-    GITHUB_DEPLOYMENTS_NICE = "https://github.com/balancer/balancer-deployments/blob/master"
-    GITHUB_RAW_OUTPUTS = "https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/outputs"
-    ZERO_ADDRESS = "0x0000000000000000000000000000000000000000"
-    CHAIN_IDS_BY_NAME = chains["CHAIN_IDS_BY_NAME"]
-    SCANNERS_BY_CHAIN = chains["SCANNERS_BY_CHAIN"]
 
     fullbook = requests.get(f"{GITHUB_RAW_OUTPUTS}/addressbook.json").json()
-    fx_description_by_name = requests.get("https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/extras/func_desc_by_name.json").json
-
-    ### Errors
-    class MultipleMatchesError(Exception):
-        pass
-
-    class NoResultError(Exception):
-        pass
+    chains = requests.get(
+        "https://raw.githubusercontent.com/BalancerMaxis/bal_addresses/main/extras/chains.json"
+    ).json()
 
     def __init__(self, chain, jsonfile=False):
-        self.jsonfile=jsonfile
+        self.jsonfile = jsonfile
         self.chain = chain
         self.dotmap = self.build_dotmap()
-        deployments = requests.get(f"{self.GITHUB_RAW_OUTPUTS}/deployments.json").json()
+        deployments = requests.get(f"{GITHUB_RAW_OUTPUTS}/deployments.json").json()
         try:
-            dold =  deployments["old"][chain]
-        except:
+            dold = deployments["old"][chain]
+        except Exception:
             dold = {}
         try:
             dactive = deployments["active"][chain]
-        except:
+        except Exception:
             dactive = {}
         self.deployments_only = DotMap(dactive | dold)
         try:
-            self.flatbook = requests.get(f"{self.GITHUB_RAW_OUTPUTS}/{chain}.json").json()
-            self.reversebook = DotMap(requests.get(f"{self.GITHUB_RAW_OUTPUTS}/{chain}_reverse.json").json())
-        except:
-            self.flatbook = {"zero/zero": self.ZERO_ADDRESS }
-            self.reversebook = {self.ZERO_ADDRESS: "zero/zero"}
+            self.flatbook = requests.get(f"{GITHUB_RAW_OUTPUTS}/{chain}.json").json()
+            self.reversebook = DotMap(
+                requests.get(f"{GITHUB_RAW_OUTPUTS}/{chain}_reverse.json").json())
+        except Exception:
+            self.flatbook = {"zero/zero": ZERO_ADDRESS}
+            self.reversebook = {ZERO_ADDRESS: "zero/zero"}
 
+        self._deployments = None
+
+    @property
+    def deployments(self) -> Optional[Munch]:
+        """
+        Get the deployments for all chains in a form of a Munch object
+        """
+        if self._deployments is not None:
+            return self._deployments
+        else:
+            self.populate_deployments()
+
+        return self._deployments
+
+    def populate_deployments(self) -> None:
+        chain_deployments = requests.get(
+            f"{GITHUB_DEPLOYMENTS_RAW}/addresses/{self.chain}.json"
+        )
+        if chain_deployments.ok:
+            self._deployments = Munch()
+            # Remove date from key
+            processed_deployment = self._process_deployment(chain_deployments.json())
+            self._deployments = Munch.fromDict(processed_deployment)
+
+    def _process_deployment(self, deployment: Dict) -> Dict:
+        """
+        Process deployment to remove date from key and replace - with _
+        """
+        processed_deployment = {}
+        for k, v in deployment.items():
+            # lstrip date in format YYYYMMDD-:
+            # Change all - to underscores
+            deployment_identifier = k.lstrip("0123456789-").replace("-", "_")
+            # Flatten contracts list to dict with name as key
+            if isinstance(v.get('contracts'), list):
+                v['contracts'] = {contract['name']: contract for contract in v['contracts']}
+            processed_deployment[deployment_identifier] = v
+        return processed_deployment
 
     def search_unique(self, substr):
         results = [s for s in self.flatbook.keys() if substr in s]
         if len(results) > 1:
-            raise self.MultipleMatchesError(f"{substr} Multiple matches found: {results}")
-        if  len(results) < 1:
-            raise self.NoResultError(f"{substr}")
+            raise MultipleMatchesError(f"{substr} Multiple matches found: {results}")
+        if len(results) < 1:
+            raise NoResultError(f"{substr}")
         return results[0]
 
     def search_many(self, substr):
@@ -76,12 +113,11 @@ def latest_contract(self, contract_name):
             if contract_name in contractData.keys():
                 deployments.append(deployment)
         if len(deployments) == 0:
-            raise self.NoResultError(contract_name)
+            raise NoResultError(contract_name)
         deployments.sort(reverse=True)
         return self.deployments_only[deployments[0]][contract_name]
 
-
-    def checksum_address_dict(addresses):
+    def checksum_address_dict(self, addresses):
         """
         convert addresses to their checksum variant taken from a (nested) dict
         """
@@ -101,8 +137,9 @@ def build_dotmap(self):
                 fullbook = json.load(f)
         else:
             fullbook = self.fullbook
-        return(DotMap(fullbook["active"].get(self.chain, {}) | fullbook["old"].get(self.chain, {})))
-        ### Checksum one more time for good measure
+        return (
+            DotMap(fullbook["active"].get(self.chain, {}) | fullbook["old"].get(self.chain, {})))
+        # Checksum one more time for good measure
 
     def flatten_dict(self, d, parent_key='', sep='/'):
         items = []
@@ -116,14 +153,11 @@ def flatten_dict(self, d, parent_key='', sep='/'):
 
     def generate_flatbook(self):
         print(f"Generating Addressbook for {self.chain}")
-        monorepo_addresses = {}
-        dupContracts = {}
         ab = dict(self.dotmap)
-        return(self.flatten_dict(ab))
-
+        return self.flatten_dict(ab)
 
 
-#  Version outside class to allow for recursion on the uninitialized class
+# Version outside class to allow for recursion on the uninitialized class
 def checksum_address_dict(addresses):
     """
     convert addresses to their checksum variant taken from a (nested) dict
@@ -136,4 +170,4 @@ def checksum_address_dict(addresses):
             checksummed[k] = checksum_address_dict(v)
         else:
             print(k, v, "formatted incorrectly")
-    return checksummed
+    return checksummed

bal_addresses/exceptions.py

@@ -0,0 +1,6 @@
+class MultipleMatchesError(Exception):
+    pass
+
+
+class NoResultError(Exception):
+    pass

bal_addresses/requirements-dev.txt

@@ -0,0 +1,4 @@
+pytest-mock
+responses
+pytest-cov
+pytest==7.4.0

bal_addresses/requirements.txt

@@ -2,4 +2,5 @@ pathlib>=1.0
 requests
 pandas
 web3==5.31.3
-dotmap
+dotmap
+munch==4.0.0

tests/test_addresses.py

@@ -0,0 +1,105 @@
+import pytest
+import responses
+
+from bal_addresses import AddrBook
+
+
+@responses.activate
+def test_deployments_populated():
+    responses.add(
+        responses.GET,
+        "https://raw.githubusercontent.com/BalancerMaxis"
+        "/bal_addresses/main/outputs/deployments.json",
+        json={
+            "BFactory": "0x9424B1412450D0f8Fc2255FAf6046b98213B76Bd",
+        }
+    )
+    responses.add(
+        responses.GET,
+        "https://raw.githubusercontent.com/balancer"
+        "/balancer-deployments/master/addresses/mainnet.json",
+        json={
+            "20210418-vault": {
+                "contracts": [
+                    {
+                        "name": "Vault",
+                        "address": "0xBA12222222228d8Ba445958a75a0704d566BF2C8"
+                    },
+                    {
+                        "name": "BalancerHelpers",
+                        "address": "0x5aDDCCa35b7A0D07C74063c48700C8590E87864E"
+                    },
+                    {
+                        "name": "ProtocolFeesCollector",
+                        "address": "0xce88686553686DA562CE7Cea497CE749DA109f9F"
+                    }
+                ],
+                "status": "ACTIVE"
+            }
+        }
+    )
+    a = AddrBook("mainnet")
+
+    a.populate_deployments()
+    assert a.deployments.vault.status == "ACTIVE"
+    assert a.deployments.vault.contracts.Vault.name == "Vault"
+    assert (
+        a.deployments.vault.contracts.Vault.address == "0xBA12222222228d8Ba445958a75a0704d566BF2C8"
+    )
+    assert a.deployments.vault.contracts.BalancerHelpers.name == "BalancerHelpers"
+    # Make sure that when we try to access a non-existing attribute, we get an error
+    with pytest.raises(AttributeError):
+        assert a.deployments.vault.non_existing_attribute
+
+
+@responses.activate
+def test_deployments_invalid_format():
+    """
+    Make sure that library is data agnostic and can handle different formats
+    """
+    responses.add(
+        responses.GET,
+        "https://raw.githubusercontent.com/BalancerMaxis"
+        "/bal_addresses/main/outputs/deployments.json",
+        json={
+            "BFactory": "0x9424B1412450D0f8Fc2255FAf6046b98213B76Bd",
+        }
+    )
+    responses.add(
+        responses.GET,
+        "https://raw.githubusercontent.com/balancer"
+        "/balancer-deployments/master/addresses/mainnet.json",
+        json={
+            "20210418-vault": {
+                "contracts": {'name': 'Vault'},
+                "status": "ACTIVE"
+            }
+        }
+    )
+    a = AddrBook("mainnet")
+
+    a.populate_deployments()
+    assert a.deployments.vault.contracts.name == "Vault"
+
+
+@responses.activate
+def test_deployments_not_populated():
+    responses.add(
+        responses.GET,
+        "https://raw.githubusercontent.com/BalancerMaxis"
+        "/bal_addresses/main/outputs/deployments.json",
+        json={
+            "BFactory": "0x9424B1412450D0f8Fc2255FAf6046b98213B76Bd",
+        }
+    )
+    responses.add(
+        responses.GET,
+        "https://raw.githubusercontent.com/balancer"
+        "/balancer-deployments/master/addresses/mainnet.json",
+        json={},
+        status=404
+    )
+    a = AddrBook("mainnet")
+    assert a.deployments is None
+    with pytest.raises(AttributeError):
+        assert a.deployments.vault.non_existing_attribute