diff --git a/deploy/show-logs.sh b/deploy/show-logs.sh index b24c23e1..005de29d 100755 --- a/deploy/show-logs.sh +++ b/deploy/show-logs.sh @@ -1,5 +1,6 @@ journalctl --follow -o cat -u backend.service \ -u bot.service \ + -u quiz.service \ -u huey.service \ -u nginx \ -u ngrok.service \ diff --git a/src/mainframe/clients/devices.py b/src/mainframe/clients/devices.py index 7ecca749..bae32c8a 100644 --- a/src/mainframe/clients/devices.py +++ b/src/mainframe/clients/devices.py @@ -62,12 +62,6 @@ def run(self): for device in devices if device.mac not in list(map(attrgetter("mac"), existing_devices)) ] - self.logger.info( - "Got %d devices%s", - len(devices), - f" ({len(new_devices)} new ones)" if new_devices else "", - extra={"new_devices": new_devices}, - ) active_macs = list(map(attrgetter("mac"), devices)) went_online = [ @@ -80,6 +74,16 @@ def run(self): for device in existing_devices if device.is_active and device.mac not in active_macs ] + self.logger.info( + "Got %d devices%s", + len(devices), + f" ({len(new_devices)} new ones)" if new_devices else "", + extra={ + "new_devices": new_devices, + "went_online": went_online, + "went_offline": went_offline, + }, + ) if went_offline: Device.objects.filter( mac__in=map(attrgetter("mac"), went_offline) diff --git a/src/mainframe/finance/management/commands/__init__.py b/src/mainframe/finance/management/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/mainframe/finance/management/commands/import_stock_pnl.py b/src/mainframe/finance/management/commands/import_stock_pnl.py deleted file mode 100644 index 94853d7b..00000000 --- a/src/mainframe/finance/management/commands/import_stock_pnl.py +++ /dev/null @@ -1,61 +0,0 @@ -import csv -from datetime import datetime -from pathlib import Path - -from django.conf import settings -from django.core.exceptions import ValidationError -from django.core.management.base import BaseCommand -from django.db import IntegrityError -from mainframe.clients.chat import send_telegram_message -from mainframe.clients.logs import get_default_logger -from mainframe.finance.models.stocks import PnL -from mainframe.finance.tasks import backup_finance_model - - -def parse_pnl(file_name): - def normalize_key(key): - key = key.lower().replace(" ", "_") - if key == "symbol": - return "ticker" - if key == "realised_pnl": - return "pnl" - return key - - with open(file_name) as file: - reader = csv.DictReader(file) - return [ - PnL(**{normalize_key(k): v for (k, v) in row.items()}) for row in reader - ] - - -class Command(BaseCommand): - def handle(self, *_, **options): - logger = get_default_logger(__name__) - - logger.info("Importing stock PnL") - now = datetime.now() - - total = 0 - data_path = settings.BASE_DIR / "finance" / "data" / "stock_pnl" - - for file_name in Path(data_path).glob("**/*.csv"): - logger.info("Parsing %s", file_name.name) - try: - results = PnL.objects.bulk_create( - parse_pnl(file_name), ignore_conflicts=True - ) - except (IntegrityError, ValidationError) as e: - logger.error(str(e)) - file_name.rename(f"{file_name}.{e}.{now}.failed") - continue - else: - results_count = len(results) - logger.info("%s: %d rows", file_name.stem, results_count) - total += results_count - logger.info("Import completed - Deleting %s", file_name.stem) - file_name.unlink() - - msg = f"Imported {total} stock PnL" - send_telegram_message(text=msg) - if total: - backup_finance_model(model="PnL") diff --git a/src/mainframe/finance/management/commands/import_stock_transactions.py b/src/mainframe/finance/management/commands/import_stock_transactions.py deleted file mode 100644 index 5aed6f23..00000000 --- a/src/mainframe/finance/management/commands/import_stock_transactions.py +++ /dev/null @@ -1,71 +0,0 @@ -import csv -from datetime import datetime -from pathlib import Path - -from django.conf import settings -from django.core.exceptions import ValidationError -from django.core.management.base import BaseCommand -from django.db import IntegrityError -from mainframe.clients.chat import send_telegram_message -from mainframe.clients.logs import get_default_logger -from mainframe.finance.models.stocks import StockTransaction -from mainframe.finance.tasks import backup_finance_model - - -def normalize_row(row: dict): - def normalize_key(key): - return key.lower().replace(" ", "_") - - def normalize_value(value): - return str(value).replace("$", "").replace("€", "") - - row["Type"] = normalize_type(row["Type"]) - return {normalize_key(k): normalize_value(v) for (k, v) in row.items() if v} - - -def normalize_type(stock_type): - types = dict(StockTransaction.TYPE_CHOICES).values() - if stock_type not in types: - return StockTransaction.TYPE_OTHER - return list(types).index(stock_type) + 1 - - -def parse_transactions(file_name, _): - with open(file_name) as file: - reader = csv.DictReader(file) - return [StockTransaction(**normalize_row(row)) for row in reader] - - -class Command(BaseCommand): - def handle(self, *_, **options): - logger = get_default_logger(__name__) - - logger.info("Importing stock statements") - now = datetime.now() - - total = 0 - data_path = settings.BASE_DIR / "finance" / "data" / "stock_transactions" - - for file_name in Path(data_path).glob("**/*.csv"): - logger.info("Parsing %s", file_name.name) - transactions = parse_transactions(file_name, logger) - - try: - results = StockTransaction.objects.bulk_create( - transactions, ignore_conflicts=True - ) - except (IntegrityError, ValidationError) as e: - logger.error(str(e)) - file_name.rename(f"{file_name}.{e}.{now}.failed") - continue - else: - results_count = len(results) - logger.info("%s: %d rows", file_name.stem, results_count) - total += results_count - logger.info("Import completed - Deleting %s", file_name.stem) - file_name.unlink() - - msg = f"Imported {total} stock transactions" - send_telegram_message(text=msg) - if total: - backup_finance_model(model="StockTransaction") diff --git a/src/mainframe/finance/migrations/0033_pnl_country_pnl_isin_pnl_security_name_and_more.py b/src/mainframe/finance/migrations/0033_pnl_country_pnl_isin_pnl_security_name_and_more.py new file mode 100644 index 00000000..969ca8dc --- /dev/null +++ b/src/mainframe/finance/migrations/0033_pnl_country_pnl_isin_pnl_security_name_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 5.0.4 on 2024-11-12 17:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("finance", "0032_alter_account_type"), + ] + + operations = [ + migrations.AddField( + model_name="pnl", + name="country", + field=models.CharField(default="US", max_length=2), + preserve_default=False, + ), + migrations.AddField( + model_name="pnl", + name="isin", + field=models.CharField(default="default", max_length=12), + preserve_default=False, + ), + migrations.AddField( + model_name="pnl", + name="security_name", + field=models.CharField(default="default", max_length=50), + preserve_default=False, + ), + migrations.AddConstraint( + model_name="pnl", + constraint=models.UniqueConstraint( + fields=("date_acquired", "date_sold", "ticker", "quantity", "currency"), + name="finance_pnl_date_acquired_date_sold_ticker_quantity_currency_uniq", + ), + ), + ] diff --git a/src/mainframe/finance/models/stocks.py b/src/mainframe/finance/models/stocks.py index 7700b602..c1c384e0 100644 --- a/src/mainframe/finance/models/stocks.py +++ b/src/mainframe/finance/models/stocks.py @@ -5,14 +5,24 @@ class PnL(TimeStampedModel): amount = models.DecimalField(decimal_places=2, max_digits=7) cost_basis = models.DecimalField(decimal_places=2, max_digits=6) + country = models.CharField(max_length=2) currency = models.CharField(max_length=3) date_acquired = models.DateField() date_sold = models.DateField() + isin = models.CharField(max_length=12) pnl = models.DecimalField(decimal_places=2, max_digits=7) quantity = models.DecimalField(decimal_places=8, max_digits=11) + security_name = models.CharField(max_length=50) ticker = models.CharField(blank=True, max_length=5) class Meta: + constraints = ( + models.UniqueConstraint( + name="%(app_label)s_%(class)s_" + "date_acquired_date_sold_ticker_quantity_currency_uniq", + fields=("date_acquired", "date_sold", "ticker", "quantity", "currency"), + ), + ) ordering = ["-date_sold", "-date_acquired", "ticker"] def __str__(self): diff --git a/src/mainframe/finance/viewsets/stocks.py b/src/mainframe/finance/viewsets/stocks.py index 1201d0e8..33e1f90e 100644 --- a/src/mainframe/finance/viewsets/stocks.py +++ b/src/mainframe/finance/viewsets/stocks.py @@ -1,8 +1,15 @@ from django.db.models import Count, Q, Sum +from mainframe.clients.finance.stocks import ( + StockImportError, + StockPnLImporter, + StockTransactionsImporter, +) +from mainframe.clients.logs import get_default_logger from mainframe.finance.models import PnL, StockTransaction from mainframe.finance.serializers import PnLSerializer, StockTransactionSerializer -from rest_framework import viewsets +from rest_framework import status, viewsets from rest_framework.permissions import IsAdminUser +from rest_framework.response import Response class PnLViewSet(viewsets.ModelViewSet): @@ -10,6 +17,16 @@ class PnLViewSet(viewsets.ModelViewSet): queryset = PnL.objects.all() serializer_class = PnLSerializer + def create(self, request, *args, **kwargs): + file = request.FILES["file"] + logger = get_default_logger(__name__) + try: + StockPnLImporter(file, logger).run() + except StockImportError as e: + logger.error("Could not process file: %s - error: %s", file, e) + return Response(f"Invalid file: {file}", status.HTTP_400_BAD_REQUEST) + return self.list(request, *args, **kwargs) + def get_queryset(self): queryset = super().get_queryset() if currency := self.request.query_params.getlist("currency"): @@ -50,6 +67,16 @@ class StocksViewSet(viewsets.ModelViewSet): queryset = StockTransaction.objects.all() serializer_class = StockTransactionSerializer + def create(self, request, *args, **kwargs): + file = request.FILES["file"] + logger = get_default_logger(__name__) + try: + StockTransactionsImporter(file, logger).run() + except StockImportError as e: + logger.error("Could not process file: %s - error: %s", file, e) + return Response(f"Invalid file: {file}", status.HTTP_400_BAD_REQUEST) + return self.list(request, *args, **kwargs) + def get_queryset(self): queryset = super().get_queryset() if currency := self.request.query_params.getlist("currency"):