Revert "Delete More Stuff (#52)"

This reverts commit fb7dff8.

.env.sample

@@ -1,3 +1,10 @@
+VOLUME_PATH=data
+APP_DATA_MAX_RETRIES=3
+APP_DATA_GIVE_UP_THRESHOLD=100
+
+# Dune credentials
+DUNE_API_KEY=
+
 # AWS Credentials
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
@@ -6,3 +13,10 @@ AWS_INTERNAL_ROLE=
 AWS_EXTERNAL_ROLE=
 AWS_EXTERNAL_ID=
 AWS_BUCKET=
+
+#Orderbook DB Credentials
+BARN_DB_URL={user}:{password}@{host}:{port}/{database}
+PROD_DB_URL={user}:{password}@{host}:{port}/{database}
+
+# IPFS Gateway
+IPFS_ACCESS_TOKEN=

.github/workflows/deploy.yaml

@@ -0,0 +1,34 @@
+name: deploy
+
+on:
+  push:
+    branches: [main]
+    tags: [v*]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: ghcr.io/${{ github.repository }}
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/pull-request.yaml

@@ -24,3 +24,29 @@ jobs:
         run: black --check ./
       - name: Type Check (mypy)
         run: mypy src --strict
+
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Requirements
+        run: pip install -r requirements/dev.txt
+      - name: Unit Tests
+        run: python -m pytest tests/unit
+        env:
+          IPFS_ACCESS_KEY: ${{ secrets.IPFS_ACCESS_KEY }}
+      - name: Integration Tests
+        run: python -m pytest tests/integration
+        env:
+          PROD_DB_URL: ${{ secrets.PROD_DB_URL }}
+          BARN_DB_URL: ${{ secrets.BARN_DB_URL }}
+          WAREHOUSE_URL: ${{ secrets.WAREHOUSE_URL }}

Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.10
+
+WORKDIR /app
+
+COPY requirements/* requirements/
+RUN pip install -r requirements/prod.txt
+COPY ./src ./src
+COPY logging.conf .
+
+ENTRYPOINT [ "python3", "-m" , "src.main"]

requirements/dev.txt

@@ -5,4 +5,5 @@ black>=22.6.0
 mypy==1.3.0
 mypy-extensions==1.0.0
 pylint>=2.14.4
-pytest>=7.1.2
+pytest>=7.1.2
+sqlalchemy-stubs>=0.4

requirements/prod.txt

@@ -1,5 +1,9 @@
 dune-client>=0.3.0
+psycopg2-binary>=2.9.3
 python-dotenv>=0.20.0
 requests>=2.28.1
 pandas>=1.5.0
-boto3>=1.26.12
+ndjson>=0.3.1
+py-multiformats-cid>=0.4.4
+boto3>=1.26.12
+SQLAlchemy<2.0

src/dune_queries.py

@@ -0,0 +1,44 @@
+"""
+Localized account of all Queries related to this project's main functionality
+"""
+from __future__ import annotations
+
+from copy import copy
+from dataclasses import dataclass
+
+from dune_client.query import Query
+from dune_client.types import QueryParameter
+
+
+@dataclass
+class QueryData:
+    """Stores name and a version of the query for each query."""
+
+    name: str
+    query: Query
+
+    def __init__(self, name: str, query_id: int, filename: str) -> None:
+        self.name = name
+        self.filepath = filename
+        self.query = Query(query_id, name)
+
+    def with_params(self, params: list[QueryParameter]) -> Query:
+        """
+        Copies the query and adds parameters to it, returning the copy.
+        """
+        # We currently default to the V1 Queries, soon to switch them out.
+        query_copy = copy(self.query)
+        query_copy.params = params
+        return query_copy
+
+
+QUERIES = {
+    "APP_HASHES": QueryData(
+        query_id=1610025, name="Unique App Hashes", filename="app_hashes.sql"
+    ),
+    "LATEST_APP_HASH_BLOCK": QueryData(
+        query_id=1615490,
+        name="Latest Possible App Hash Block",
+        filename="app_hash_latest_block.sql",
+    ),
+}

src/environment.py

@@ -2,4 +2,5 @@
 from pathlib import Path
 
 PROJECT_ROOT = Path(__file__).parent.parent
+QUERY_PATH = PROJECT_ROOT / Path("src/sql")
 LOG_CONFIG_FILE = PROJECT_ROOT / Path("logging.conf")

src/fetch/dune.py

@@ -0,0 +1,85 @@
+"""
+All Dune Query executions should be routed through this file.
+TODO - Move reusable components into dune-client:
+    https://github.com/cowprotocol/dune-bridge/issues/40
+"""
+import asyncio
+import sys
+
+from dune_client.client import DuneClient
+from dune_client.query import Query
+from dune_client.types import DuneRecord
+from requests import HTTPError
+
+from src.dune_queries import QUERIES
+from src.logger import set_log
+from src.models.block_range import BlockRange
+
+log = set_log(__name__)
+
+
+class DuneFetcher:
+    """
+    Class containing, DuneClient, FileIO and a logger for convenient Dune Fetching.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+    ) -> None:
+        """
+        Class constructor.
+        Builds DuneClient from `api_key` along with a logger and FileIO object.
+        """
+        self.dune = DuneClient(api_key)
+
+    async def fetch(self, query: Query) -> list[DuneRecord]:
+        """Async Dune Fetcher with some exception handling."""
+        log.debug(f"Executing {query}")
+
+        try:
+            # Tried to use the AsyncDuneClient, without success:
+            # https://github.com/cowprotocol/dune-client/pull/31#issuecomment-1316045313
+            response = await asyncio.to_thread(
+                self.dune.refresh, query, ping_frequency=10
+            )
+            if response.state.is_complete():
+                response_rows = response.get_rows()
+                log.debug(
+                    f"Got {len(response_rows)} results for execution {response.execution_id}"
+                )
+                return response_rows
+
+            message = (
+                f"query execution {response.execution_id} incomplete {response.state}"
+            )
+            log.error(message)
+            raise RuntimeError(f"no results for {message}")
+        except HTTPError as err:
+            log.error(f"Got {err} - Exiting")
+            sys.exit()
+
+    async def latest_app_hash_block(self) -> int:
+        """
+        Block Range is used to app hash fetcher where to find the new records.
+        block_from: read from file `fname` as a loaded singleton.
+            - uses genesis block is no file exists (should only ever happen once)
+            - raises RuntimeError if column specified does not exist.
+        block_to: fetched from Dune as the last indexed block for "GPv2Settlement_call_settle"
+        """
+        return int(
+            # KeyError here means the query has been modified and column no longer exists
+            # IndexError means no results were returned from query (which is unlikely).
+            (await self.fetch(QUERIES["LATEST_APP_HASH_BLOCK"].query))[0][
+                "latest_block"
+            ]
+        )
+
+    async def get_app_hashes(self, block_range: BlockRange) -> list[DuneRecord]:
+        """
+        Executes APP_HASHES query for the given `block_range` and returns the results
+        """
+        app_hash_query = QUERIES["APP_HASHES"].with_params(
+            block_range.as_query_params()
+        )
+        return await self.fetch(app_hash_query)