Merge pull request #17 from timewave-computer/feature-opp-execution

UX Improvements, Performance Improvements, Bug Fixes
timewave-computer · Jun 18, 2024 · eb1a417 · eb1a417
2 parents 5dbbcd1 + cb10c00
commit eb1a417
Show file tree

Hide file tree

Showing 56 changed files with 11,689 additions and 871 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -6,35 +6,15 @@ jobs:
   test:
     name: python test
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.11", "3.12"]
-
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: pip install -r requirements.txt
-      - name: Run unit tests
-        run: python -m pytest tests
-      - name: Check src mypy
-        run: mypy src --install-types --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs --strict
-      - name: Check tests mypy
-        run: mypy tests --install-types --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs --strict
-      - name: Check main mypy
-        run: mypy main.py --install-types --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs --strict
-      - name: Check linter
-        run: ruff check src
-      - name: Check linter tests
-        run: ruff check tests
-      - name: Check linter main
-        run: ruff check main.py
-      - name: Check linter (pylint)
-        run: python -m pylint src
-      - name: Check linter tests (pylint)
-        run: python -m pylint tests
-      - name: Check linter main (pylint)
-        run: python -m pylint main.py
+      - name: Install nix
+        uses: cachix/install-nix-action@v27
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Compile protobuf
+        run: nix develop --command make proto
+      - name: Lint
+        run: nix develop --command make lint
+      - name: Run tests
+        run: nix develop --command make test
+
diff --git a/.gitignore b/.gitignore
@@ -162,4 +162,10 @@ cython_debug/
 # Editor files
 *~
 \#*\#
-.\#*
+.\#*
+
+# Protobuf osmosis stubs
+build/gen/osmosis
+
+# Arb bot configs
+net_config.json
diff --git a/Makefile b/Makefile
@@ -0,0 +1,30 @@
+PROTO_DIR = 'proto'
+PROTO_SOURCES := $(shell find $(PROTO_DIR) -name '*.proto')
+
+build/gen:
+	mkdir -p $@
+
+.PHONY: proto
+proto: build/gen $(PROTO_SOURCES)
+	protoc --proto_path=$(PROTO_DIR) --python_out=build/gen --mypy_out=build/gen --go-grpc_out=build/gen --mypy_grpc_out=build/gen $(PROTO_SOURCES)
+
+.PHONY: test
+test:
+	PYTHONPATH=src:build/gen python -m pytest tests
+
+.PHONY: run
+run:
+	PYTHONPATH=src:build/gen python main.py
+
+.PHONY: lint
+lint:
+	mypy src --install-types --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs --strict
+	mypy tests --install-types --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs --strict
+	mypy main.py --install-types --disallow-untyped-calls --disallow-untyped-defs --disallow-incomplete-defs
+	ruff check src
+	ruff check tests
+	ruff check main.py
+
+.PHONY: clean
+clean:
+	rm -rf build/gen
diff --git a/README.md b/README.md
@@ -4,6 +4,45 @@ An extensible arbitrage bot for trading against valence auctions and Astroport a
 
 ## Installation
 
+### Nix
+
+Use [Nix](https://github.com/NixOS/nix) to automatically install all required dependencies, or [manually](#manual) install dependencies.
+
+Nix can be installed through the following steps on MacOS and Linux:
+
+#### MacOS
+
+```sh
+curl -L https://nixos.org/nix/install | sh
+```
+
+#### Linux
+
+```sh
+curl -L https://nixos.org/nix/install | sh -s -- --daemon
+```
+
+After installing Nix, enter the abritrage bot dev environment by executing
+
+```sh
+nix develop --extra-experimental-features nix-command --extra-experimental-features flakes
+```
+
+in the repository root. Proceed to [Usage](#usage) to use the arbitrage bot.
+
+### Manual
+
+#### Dependencies
+
+This tool requires the following to be installed:
+
+* Python 3.11
+* `protoc`
+* `protoc-gen-grpc`
+* `make`
+
+#### Procedure
+
 Installation requires Python 3.11.
 
 To install the auction-arbitrage-bot, first create a virtual environment:
@@ -24,50 +63,65 @@ Then, install the required dependencies:
 pip install -r requirements.txt
 ```
 
+Then, compile cosmos and osmosis protobuf stubs:
+
+```sh
+make proto
+```
+
 ## Usage
 
-The auction arbitrage bot can be run with only one required flag: `wallet_address`. This flag specifies where to look for initial funds for arbitrage. All available flags are:
+The auction arbitrage bot can be run with only one required environment variable: `WALLET_MNEMONIC`. This environment variable specifies where to look for initial funds for arbitrage. All available flags are:
 
 * `-f` (`--pool_file`): Specifies which pools to use for the Neutron Astroport and Osmosis pool providers, and which routes to use for the Scheduler. Can also be used to cache requests required to obtain pool information.
 * `-p` (`--poll_interval`): Specifies how frequently the arbitrage strategy should be run (in seconds). The default value is `120`.
 * `-d` (`--discovery_interval`): Specifies how frequently new arbitrage routes should be discovered (in seconds). The default value is `600`.
-* `-m` (`--max_hops`): Specifies the maximum number of "hops" in a single arbitrage trade. The default value is `3`. Note that increasing this value increases the time for the discovery algorithm to run.
-* `n` (`--num_routes_considered`): Specifies the number of routes to discover. The default value is `30`. Note that increasing this value increases the time for the discovery algorithm to run.
+* `-nh` (`--hops`): Specifies the number of "hops" in a single arbitrage trade. The default value is `3`. Note that increasing this value increases the time for the discovery algorithm to run.
+* `-r` (`--require_leg_types`): Specifies the required arbitrage pool types to require trades include. Possible values include: auction osmosis astroport. Multiple leg types may be required, separated by spaces.
 * `-b` (`--base_denom`): Specifies the denom in which profits are denominated. The default value is the Neutron Noble USDC denom (`ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81`)
-* `-pm` (`--profit_margin`): Specifies the quantity of the base denom that must be obtained from an arbitrage opportunity to consider it. The default value is `10`.
-* `-w` (`--wallet_address`): Specifies the address of the wallet from which funds should be used to execute trades. This flag is **required**.
+* `-pm` (`--profit_margin`): Specifies the quantity of the base denom that must be obtained from an arbitrage opportunity to consider it. The default value is `1000`.
+* `-l` (`--log_file`): Specifies a file to write daemon logs to. If left unspecified, stdout will be used. Stdout and a log file cannot be used simultaneously.
 * `-c` (`--net_config`): Specifies a path to a file containing RPC URL's to use for different networks.
 
 The bot may be run by executing:
 
 ```sh
-python main.py --wallet_address <WALLET_ADDRESS>
+WALLET_MNEMONIC=<wallet_mnemonic> python main.py
+```
+
+View available flags by executing:
+
+```sh
+python main.py --help
 ```
 
 ### Commands
 
 The bot may also be supplied a command (an argument with no hyphens). The available commands are as follows:
 
-* `dump`: Explores all known pools with the given `--max_hops` and `--limit`, and writes discovered routes to a file, exiting immediately after. Results will be written in JSON format, following conventions outlined below.
-  * Sample execution: `python main.py --wallet_address <WALLET_ADDRESS> dump`
+* `dump`: Loads all known pools, and writes discovered pools to a file, exiting immediately after. Results will be written in JSON format, following conventions outlined below.
+  * Sample execution: `WALLET_MNEMONIC=<wallet_mnemonic> python main.py dump`
+* `daemon`: Spawns the searcher event-loop as a long-running background "daemon" process, after initial setup. Exits the init script after spawning the daemon, leaving the daemon running in the background.
+* `hist`: Shows summary statistics for the bot's execution, including a lot of all recently completed orders.
+* `hist show <id>`: Shows execution logs for a particular order, as well as the execution plan for the order.
 
 ### Custom RPC's
 
 Custom RPC providers may be specified with the `--net_config` flag. This flag specifies a path to a JSON file with the following format:
 
 ```json
 {
-  "neutron": [
-    "grpc+https://neutron-grpc.publicnode.com:443"
-  ],
-  "osmosis": [
-    "https://lcd.osmosis.zone"
-  ]
+  "neutron": {
+    "http": ["https://neutron-rest.publicnode.com"],
+	"grpc": ["grpc+https://neutron-grpc.publicnode.com:443"]
+  },
+  "osmosis": {
+    "http": ["https://lcd.osmosis.zone"],
+	"grpc": ["https://osmosis-rpc.publicnode.com:443"]
+  }
 }
 ```
 
-Currently, only HTTP/REST endpoints are supported for Osmosis.
-
 ### Custom Pools
 
 Custom pools and routes may be provided by utilizing a "pool file" via the `--pool_file` flag. Using this flag will avoid external calls to obtain pool listings, and will result in calls to `Directory` class' `.pools` methods looking up available pools from this file. Furthermore, if routes are specified, no route discovery will be performed, and the routes provided will be used. An example pool file is as follows:
@@ -93,8 +147,9 @@ Custom pools and routes may be provided by utilizing a "pool file" via the `--po
     "osmosis": [
       {
 	    "asset_a": "<DENOM>",
-	    "asset_b": "<DENOM>,
-	    "pool_id": 1234
+	    "asset_b": "<DENOM>",
+	    "pool_id": 1234,
+		"address": "<ADDR>"
 	  }
     ]
   },
@@ -133,8 +188,8 @@ Custom pools and routes may be provided by utilizing a "pool file" via the `--po
 	    "auction": {
 	      "asset_a": "<DENOM>",
 	      "asset_b": "<DENOM>",
-	      "address": "<ADDR>"		  
-		}
+	      "address": "<ADDR>"
+        }
 	  }
 	]
   ]
@@ -143,31 +198,44 @@ Custom pools and routes may be provided by utilizing a "pool file" via the `--po
 
 ### Output logs
 
-By default, the arbitrage bot logs:
+By default, the arbitrage bot logs to the INFO level the following:
+
+* Attempted arbitrage trades
+* Successful arbitrage trades
+* Failing arbitrage trades
 
-* Discovered routes
-* Considered routes
+Debug information is logged to the DEBUG level, including:
+
+* Discovered arbitrage trades
+* Profit levels for considered trades
+* Intermediate debug logs
+
+The log level may be set via the `LOGLEVEL` environment variable. Possible values are: `INFO`, `DEBUG`, or `ERROR`.
 
 An example output is as follows:
 
 ```
 INFO:__main__:Building pool catalogue
-INFO:__main__:Built pool catalogue with 2274 pools
-INFO:src.strategies.naive:Building route tree from ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 with 4560 vertices (this may take a while)
-INFO:src.strategies.naive:Closed circuit from ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 to ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81; registering route
-INFO:src.strategies.naive:Discovered route with 3 hop(s): ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 - ibc/2CB87BCE0937B1D1DFCEE79BE4501AAF3C265E923509AEAC410AD85D27F35130 -> ibc/F082B65C88E4B6D5EF1DB243CDA1D331D002759E938A0F5CD3FFDC5D53B3E349 - ibc/2CB87BCE0937B1D1DFCEE79BE4501AAF3C265E923509AEAC410AD85D27F35130 -> ibc/F082B65C88E4B6D5EF1DB243CDA1D331D002759E938A0F5CD3FFDC5D53B3E349 - ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81
-INFO:src.strategies.naive:Closed circuit from ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 to ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81; registering route
-INFO:src.strategies.naive:Discovered route with 3 hop(s): ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 - ibc/2CB87BCE0937B1D1DFCEE79BE4501AAF3C265E923509AEAC410AD85D27F35130 -> ibc/F082B65C88E4B6D5EF1DB243CDA1D331D002759E938A0F5CD3FFDC5D53B3E349 - ibc/2CB87BCE0937B1D1DFCEE79BE4501AAF3C265E923509AEAC410AD85D27F35130 -> ibc/F082B65C88E4B6D5EF1DB243CDA1D331D002759E938A0F5CD3FFDC5D53B3E349 - ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81
-...
-INFO:src.strategies.naive:Finished building route tree; discovered 30 routes
-INFO:src.strategies.naive:Found 6 profitable routes, with max profit of 139782323 and min profit of 6223766
-INFO:src.strategies.naive:Candidate arbitrage opportunity #1 with profit of 6223766 and route with 3 hop(s): astroport: factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO - ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 -> astroport: factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO - ibc/F082B65C88E4B6D5EF1DB243CDA1D331D002759E938A0F5CD3FFDC5D53B3E349 -> astroport: ibc/F082B65C88E4B6D5EF1DB243CDA1D331D002759E938A0F5CD3FFDC5D53B3E349 - ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81
-INFO:src.strategies.naive:Candidate arbitrage opportunity #2 with profit of 6223766 and route with 3 hop(s): astroport: factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO - ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 -> astroport: factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO - ibc/F082B65C88E4B6D5EF1DB243CDA1D331D002759E938A0F5CD3FFDC5D53B3E349 -> astroport: ibc/F082B65C88E4B6D5EF1DB243CDA1D331D002759E938A0F5CD3FFDC5D53B3E349 - ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81
+INFO:__main__:Built pool catalogue with 2440 pools
+INFO:src.strategies.naive:Finding profitable routes
+INFO:src.strategies.naive:Executing route with profit of 43521870: astroport: factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO - ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 -> osmosis: ibc/73BB20AF857D1FE6E061D01CA13870872AD0C979497CAF71BEA25B1CBF6879F1 - ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4 -> valence: ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 - factory/neutron1p8d89wvxyjcnawmgw72klknr3lg9gwwl6ypxda/newt
+INFO:src.strategies.naive:Queueing candidate arbitrage opportunity with route with 3 hop(s): astroport: factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO - ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 -> osmosis: ibc/73BB20AF857D1FE6E061D01CA13870872AD0C979497CAF71BEA25B1CBF6879F1 - ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4 -> valence: ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 - factory/neutron1p8d89wvxyjcnawmgw72klknr3lg9gwwl6ypxda/newt
+INFO:src.strategies.naive:Queueing arb leg on astroport with factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO -> ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81
+INFO:src.strategies.naive:Executing arb leg on astroport with 534486 ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 -> factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO
+INFO:src.strategies.naive:Arb leg can be executed atomically; no transfer necessary
+INFO:src.strategies.naive:Submitting arb to contract: neutron15wal8wsy7mq37hagmrzchwmugpjzwlzrlw7pylkhlfuwukmc2kps722ems
+INFO:src.strategies.naive:Executed leg ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 -> factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO: 99D428AFC53499249A5FCA624F19273CCF7F886322621F79B12358830745739C
+INFO:src.strategies.naive:Queueing arb leg on osmosis with ibc/73BB20AF857D1FE6E061D01CA13870872AD0C979497CAF71BEA25B1CBF6879F1 -> ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4
+INFO:src.strategies.naive:Executing arb leg on osmosis with 4789772 ibc/73BB20AF857D1FE6E061D01CA13870872AD0C979497CAF71BEA25B1CBF6879F1 -> ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4
+INFO:src.strategies.naive:Transfering 4789772 factory/neutron154gg0wtm2v4h9ur8xg32ep64e8ef0g5twlsgvfeajqwghdryvyqsqhgk8e/APOLLO from neutron-1 -> osmosis-1
+INFO:src.strategies.naive:Submitted IBC transfer from src neutron-1 to osmosis-1: A4C71E50DC5CAE19BB12AB0F3F0417AAD86352AC60BFE51E2B270579EBDDC818
+INFO:src.strategies.naive:Checking IBC transfer status A4C71E50DC5CAE19BB12AB0F3F0417AAD86352AC60BFE51E2B270579EBDDC818
+INFO:src.strategies.naive:Transfer succeeded: neutron-1 -> osmosis-1
+INFO:src.strategies.naive:Balance to swap for osmo179y4yx6m0r73r0qlrx5axt7m7d05xpp3urkzve on osmosis-1: 4789772
+INFO:src.strategies.naive:Submitting arb to pool: 1410
+INFO:src.strategies.naive:Executed leg ibc/73BB20AF857D1FE6E061D01CA13870872AD0C979497CAF71BEA25B1CBF6879F1 -> ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4: DBD4B87C02A2EFA89125A3D8DC47FB3B3802C066AD8FAAF70EE1F1F8236F6583
+INFO:src.strategies.naive:Queueing arb leg on valence with ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 -> factory/neutron1p8d89wvxyjcnawmgw72klknr3lg9gwwl6ypxda/newt
+INFO:src.strategies.naive:Executing arb leg on valence with 513630 ibc/B559A80D62249C8AA07A380E2A2BEA6E5CA9A6F079C912C3A9E9B494105E4F81 -> factory/neutron1p8d89wvxyjcnawmgw72klknr3lg9gwwl6ypxda/newt
+INFO:src.strategies.naive:Transfering 513630 ibc/498A0751C798A0D9A389AA3691123DADA57DAA4FE165D5C75894505B876BA6E4 from osmosis-1 -> neutron-1
 ...
 ```
-
-# To Do
-
-- [ ] Add an example strategy for docs that uses threading
-- [ ] Track which pools have been touched to inform route samping, or get a price feed from coingecko, and just filter for the coins that are relevant to the DEX you're looking at and query for large price movements, and you can pick the top 10 to inform route collection
-- [ ] Look at Yan's contract and do backtesting (95% of blocks): trailing 7 days simulated number of auctions
diff --git a/contracts/auction.wasm b/contracts/auction.wasm
diff --git a/contracts/auctions_manager.wasm b/contracts/auctions_manager.wasm