Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

calculate prices from csv #96

Merged
merged 47 commits into from
Feb 5, 2022
Merged
Changes from 1 commit
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
43e1fe5
calculate prices from csv
scientes Dec 13, 2021
74077df
Merge branch 'main' into prices-from-csv
scientes Dec 13, 2021
f2729c7
fixed linting except line to long
scientes Dec 13, 2021
13fa82e
fixed for multiple coins
scientes Dec 19, 2021
9afb70a
added sql migration and extra function
scientes Dec 25, 2021
8e21d8a
bugfix (prices were inverted db) and migration fix
scientes Dec 25, 2021
9c52dd8
fixed missing inversion
scientes Dec 31, 2021
f74327d
formatting
scientes Dec 31, 2021
5112a14
Prices from csv (#3)
Griffsano Dec 31, 2021
45b6a39
FIX type annotation of group_by
provinzio Jan 2, 2022
7c0f924
REFACTOR add newline
provinzio Jan 2, 2022
dc852ee
REFACTOR get_price_from_csv
provinzio Jan 2, 2022
62b861c
ADD docstring to _sort_pair
provinzio Jan 2, 2022
5124ebe
RENAME reciprocal bool to inverted
provinzio Jan 2, 2022
3d10f3d
ADD comment in get_price when price is missing
provinzio Jan 2, 2022
30fe876
TODO ADD rudimentary patch functions
provinzio Jan 2, 2022
9ea7822
boilerplate
scientes Jan 7, 2022
e711b47
removed classes and replaced get_price occurrences
scientes Jan 9, 2022
0dd0132
refractored db functions
scientes Jan 9, 2022
33876f2
fixed patch function
scientes Jan 9, 2022
ab15eaa
fixed imports and flake added patch 001
scientes Jan 10, 2022
7265ee7
fix iosort
scientes Jan 10, 2022
7bb7bd8
Prices from csv (#5)
Griffsano Jan 28, 2022
179bb05
Refactor logging patching info
provinzio Feb 5, 2022
acc215b
CHANGE get_tablenames does not query §version table per default
provinzio Feb 5, 2022
1419746
UPDATE transfer platform parameter
provinzio Feb 5, 2022
d70559f
REFACTOR function docstrings
provinzio Feb 5, 2022
09a81e5
FIX mypy linting error
provinzio Feb 5, 2022
a06f735
Reorder preamble of set_price_db
provinzio Feb 5, 2022
e99a7aa
UPDATE assert db exists before setting new price
provinzio Feb 5, 2022
cd33058
ADD helper functions to get patch func names/versions
provinzio Feb 5, 2022
b54c366
REFACTOR patch databases
provinzio Feb 5, 2022
63b3959
ADD create database in book.py when missing
provinzio Feb 5, 2022
c400a4c
AUTOFORMAT database.py
provinzio Feb 5, 2022
78046f4
Avoid circular import
provinzio Feb 5, 2022
7b81aa9
CHANGE update_version create §version table when missing
provinzio Feb 5, 2022
700ea26
Merge remote-tracking branch 'origin/main' into prices-from-csv
provinzio Feb 5, 2022
0f2e7fb
UPDATE format of warning message
provinzio Feb 5, 2022
a6fc0eb
UPDATE reword debug message
provinzio Feb 5, 2022
7cae5c3
ADD debug message when updating db version
provinzio Feb 5, 2022
f932e0c
CHANGE Create db not in book, but on set_price when db is missing
provinzio Feb 5, 2022
0b93b3b
UPDATE duplicate price warning, add db price for comparison
provinzio Feb 5, 2022
81b4e30
AUTOFORMAT price_data
provinzio Feb 5, 2022
53753d6
ADD comment to price_data when querying new price
provinzio Feb 5, 2022
16dabab
FIX database type for price
provinzio Feb 5, 2022
c1538cf
ADD set_price_db parameter to overwrite already existing prices
provinzio Feb 5, 2022
9f906b1
FIX linting errors
provinzio Feb 5, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
TODO ADD rudimentary patch functions
provinzio committed Jan 2, 2022
commit 30fe87689b5d98833666fc97784b3dad2ea52356
2 changes: 2 additions & 0 deletions src/book.py
Original file line number Diff line number Diff line change
@@ -905,6 +905,8 @@ def read_file(self, file_path: Path) -> None:
assert file_path.is_file()

if exchange := self.detect_exchange(file_path):
# TODO check that database file exists. if missing, add file with
# highest version number (highest patch number)
try:
read_file = getattr(self, f"_read_{exchange}")
except AttributeError:
3 changes: 3 additions & 0 deletions src/main.py
Original file line number Diff line number Diff line change
@@ -18,13 +18,16 @@

import log_config # noqa: F401
from book import Book
from patch_database import patch_databases
from price_data import PriceData
from taxman import Taxman

log = logging.getLogger(__name__)


def main() -> None:
patch_databases()

price_data = PriceData()
book = Book(price_data)
taxman = Taxman(book, price_data)
174 changes: 174 additions & 0 deletions src/patch_database.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
# CoinTaxman
# Copyright (C) 2021 Carsten Docktor <https://github.com/provinzio>

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

import datetime
import decimal
import logging
import sqlite3
from pathlib import Path

import config
import misc

FUNC_PREFIX = "__patch_"
log = logging.getLogger(__name__)


def get_version(db_path: Path) -> int:
"""Get database version from a database file.

If the version table is missing, one is created.

Args:
db_path (str): Path to database file.

Raises:
RuntimeError: The database version is ambiguous.

Returns:
int: Version of database file.
"""
with sqlite3.connect(db_path) as conn:
cur = conn.cursor()
try:
cur.execute("SELECT version FROM §version;")
versions = [int(v[0]) for v in cur.fetchall()]
except sqlite3.OperationalError as e:
if str(e) == "no such table: §version":
# The §version table doesn't exist. Create one.
cur.execute("CREATE TABLE §version(version INT);")
cur.execute("INSERT INTO §version (version) VALUES (0);")
return 0
else:
raise e

if len(versions) == 1:
version = versions[0]
return version
else:
raise RuntimeError(
f"The database version of the file `{db_path.name}` is ambigious. "
f"The table `§version` should have one entry, but has {len(versions)}."
)


def update_version(db_path: Path, version: int) -> None:
with sqlite3.connect(db_path) as conn:
cur = conn.cursor()
cur.execute("TRUNCATE §version;")
assert isinstance(version, int)
cur.execute(f"INSERT INTO §version (version) VALUES ({version});")


def get_patch_func_version(func_name: str) -> int:
assert func_name.startswith(
FUNC_PREFIX
), f"Patch function `{func_name}` should start with {FUNC_PREFIX}."
len_func_prefix = len(FUNC_PREFIX)
version_str = func_name[len_func_prefix:]
version = int(version_str)
return version


def get_tablenames(cur: sqlite3.Cursor) -> list[str]:
cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
tablenames = [result[0] for result in cur.fetchall()]
return tablenames


def __patch_001(db_path: Path) -> None:
"""Convert prices from float to string

Args:
db_path (Path): [description]
"""
raise NotImplementedError


def __patch_002(db_path: Path) -> None:
"""Group tablenames, so that the symbols are alphanumerical.

Args:
db_path (Path)
"""
with sqlite3.connect(db_path) as conn:
cur = conn.cursor()
tablenames = get_tablenames(cur)

# Iterate over all tables.
for tablename in tablenames:
base_asset, quote_asset = tablename.split("/")

# Adjust the order, when the symbols aren't ordered alphanumerical.
if base_asset > quote_asset:

# Query all prices from the table.
cur = conn.execute(f"Select utc_time, price FROM `{tablename}`;")

new_values = []
for _utc_time, _price in cur.fetchall():
# Convert the data.
utc_time = datetime.datetime.strptime(
_utc_time, "%Y-%m-%d %H:%M:%S%z"
)
price = decimal.Decimal(_price)

# Calculate the price of the inverse symbol.
oth_price = misc.reciprocal(price)
new_values.append(utc_time, oth_price)

assert quote_asset < base_asset
# TODO Refactor code, so that a DatabaseHandle/Class exists,
# which presents basic functions to work with the database.
# e.g. get_tablename function from price_data
new_tablename = f"{quote_asset}/{base_asset}"
# TODO bulk insert new values in table.
# TODO Make sure, that no duplicates exists in the new table.

# Remove the old table.
cur = conn.execute(f"DROP TABLE `{tablename}`;")


def patch_databases() -> None:
# Check if any database paths exist.
database_paths = [p for p in Path(config.DATA_PATH).glob("*.db") if p.is_file()]
if not database_paths:
return

# Patch all databases separatly.
for db_path in database_paths:
# Read version from database.
current_version = get_version(db_path)

# Determine all necessary patch functions.
patch_func_names = [
func
for func in dir()
if func.startswith(FUNC_PREFIX)
if get_patch_func_version(func) > current_version
]

# Sort patch functions chronological.
patch_func_names.sort(key=get_patch_func_version)

# Run the patch functions.
for patch_func_name in patch_func_names:
patch_func = eval(patch_func_name)
patch_func(db_path)

# Update version.
new_version = get_patch_func_version(patch_func_name)
update_version(db_path, new_version)
35 changes: 4 additions & 31 deletions src/price_data.py
Original file line number Diff line number Diff line change
@@ -540,6 +540,9 @@ def __set_price_db(
utc_time (datetime.datetime)
price (decimal.Decimal)
"""
# TODO if db_path doesn't exists. Create db with §version table and
# newest version number. It would be nicer, if this could be done
# as a preprocessing step. see book.py
with sqlite3.connect(db_path) as conn:
cur = conn.cursor()
query = f"INSERT INTO `{tablename}`" "('utc_time', 'price') VALUES (?, ?);"
@@ -673,7 +676,7 @@ def get_price(

if inverted:
price = misc.reciprocal(price)
return price
return price

def get_cost(
self,
@@ -706,36 +709,6 @@ def check_database(self):
)

with sqlite3.connect(db_path) as conn:
query = "SELECT name,sql FROM sqlite_master WHERE type='table'"
cur = conn.execute(query)
for tablename, sql in cur.fetchall():
if not sql.lower().contains("price str"):
query = f"""
CREATE TABLE "sql_temp_table" (
"utc_time" DATETIME PRIMARY KEY,
"price" STR NOT NULL
);
INSERT INTO "sql_temp_table" ("price","utc_time")
SELECT "price","utc_time" FROM "{tablename}";
DROP TABLE "{tablename}";
ALTER TABLE "sql_temp_table" "{tablename}";
"""
base_asset, quote_asset = tablename.split("/")
if base_asset > quote_asset:
query = f"Select utc_time,price FROM `{tablename}`"
cur = conn.execute(query)

for row in cur.fetchall():
utc_time = datetime.datetime.strptime(
row[0], "%Y-%m-%d %H:%M:%S%z"
)
price = misc.reciprocal(decimal.Decimal(row[1]))
self.set_price_db(
platform, quote_asset, base_asset, utc_time, price
)
query = f"DROP TABLE `{tablename}`"
cur = conn.execute(query)

query = "SELECT name FROM sqlite_master WHERE type='table'"
cur = conn.execute(query)
tablenames = (result[0] for result in cur.fetchall())