Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add backend support for user flair and donations leaderboard #2978

Merged
merged 4 commits into from
Sep 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion admin/sql/create_tables.sql
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,8 @@ CREATE TABLE user_setting (
user_id INTEGER NOT NULL, --FK to "user".id
timezone_name TEXT,
troi JSONB, -- troi related prefs
brainzplayer JSONB -- brainzplayer related prefs
brainzplayer JSONB, -- brainzplayer related prefs
flair JSONB
);

ALTER TABLE user_setting
Expand Down
5 changes: 5 additions & 0 deletions admin/sql/updates/2024-09-09-add-user-flair.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
BEGIN;

ALTER TABLE user_setting ADD COLUMN flair JSONB;

COMMIT;
5 changes: 2 additions & 3 deletions consul_config.py.ctmpl
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,15 @@ REMEMBER_COOKIE_HTTPONLY = True
{{if service "pgbouncer-master"}}
{{with index (service "pgbouncer-master") 0}}
SQLALCHEMY_DATABASE_URI = "postgresql://listenbrainz:listenbrainz@{{.Address}}:{{.Port}}/listenbrainz"
MESSYBRAINZ_SQLALCHEMY_DATABASE_URI = "postgresql://messybrainz:messybrainz@{{.Address}}:{{.Port}}/messybrainz"
POSTGRES_ADMIN_URI="postgresql://postgres@{{.Address}}:{{.Port}}/postgres"
SQLALCHEMY_METABRAINZ_URI = "postgresql://metabrainz:metabrainz@{{.Address}}:{{.Port}}/metabrainz_db"
{{end}}
{{else}}
SQLALCHEMY_DATABASE_URI = "SERVICEDOESNOTEXIST_pgbouncer-master"
MESSYBRAINZ_SQLALCHEMY_DATABASE_URI = "SERVICEDOESNOTEXIST_pgbouncer-master"
POSTGRES_ADMIN_URI = "SERVICEDOESNOTEXIST_pgbouncer-master"
SQLALCHEMY_METABRAINZ_URI = "SERVICEDOESNOTEXIST_pgbouncer-master"
{{end}}


IS_MUSICBRAINZ_UP = {{template "KEY_JSON" "musicbrainz_up"}}
# Use a key '$musicbrainz_up' in consul (json boolean true or false) to decide if we connect to musicbrainz.
# If we set MB_DATABASE_URI to an empty string, then the @mb_database_needed decorator will prevent the
Expand Down
6 changes: 2 additions & 4 deletions listenbrainz/config.py.sample
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,8 @@ SQLALCHEMY_TIMESCALE_URI = "postgresql://listenbrainz_ts:listenbrainz_ts@lb_db/l
TIMESCALE_ADMIN_URI = "postgresql://postgres:postgres@lb_db/postgres"
TIMESCALE_ADMIN_LB_URI = "postgresql://postgres:postgres@lb_db/listenbrainz_ts"

# MessyBrainz
MESSYBRAINZ_SQLALCHEMY_DATABASE_URI = "postgresql://messybrainz:messybrainz@lb_db:5432/messybrainz"
MESSYBRAINZ_ADMIN_URI = "postgresql://postgres:postgres@lb_db/postgres"
MESSYBRAINZ_ADMIN_MSB_URI = "postgresql://postgres:postgres@lb_db/messybrainz"
# MetaBrainz - used for retrieving donation info from metabrainz.org
SQLALCHEMY_METABRAINZ_URI = ""
amCap1712 marked this conversation as resolved.
Show resolved Hide resolved

# MusicBrainz & others
IS_MUSICBRAINZ_UP = True # if set to False, login page will be blocked
Expand Down
181 changes: 181 additions & 0 deletions listenbrainz/db/donation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
import time
from typing import Optional

import psycopg2
import sqlalchemy
from psycopg2.extras import execute_values
from sqlalchemy import create_engine, NullPool, text

engine: Optional[sqlalchemy.engine.Engine] = None

FLAIR_MONTHLY_DONATION_THRESHOLD = 5


def init_meb_db_connection(connect_str):
"""Initializes database connection using the specified Flask app."""
global engine
while True:
try:
engine = create_engine(connect_str, poolclass=NullPool)
break
except psycopg2.OperationalError as e:
print("Couldn't establish connection to db: {}".format(str(e)))
amCap1712 marked this conversation as resolved.
Show resolved Hide resolved
print("Sleeping 2 seconds and trying again...")
time.sleep(2)


def get_flairs_for_donors(db_conn, donors):
""" Given a list of donors, add information about the user's musicbrainz username and whether the user is a listenbrainz
user and returns the updated list. """
musicbrainz_row_ids = {d.editor_id for d in donors}

query = """
SELECT u.musicbrainz_row_id
, u.musicbrainz_id
, us.flair
FROM "user" u
LEFT JOIN user_setting us
ON us.user_id = u.id
WHERE EXISTS(
SELECT 1
FROM (VALUES %s) AS t(editor_id)
WHERE u.musicbrainz_row_id = t.editor_id
)
"""
with db_conn.connection.cursor() as cursor:
results = execute_values(cursor, query, [(row_id,) for row_id in musicbrainz_row_ids], fetch=True)
lb_users = {
r[0]: {
"musicbrainz_id": r[1],
"flair": r[2]
} for r in results
}

donors_with_flair = []
for donor in donors:
donation = {
"donation": float(donor.donation),
"currency": donor.currency,
"donated_at": donor.payment_date.isoformat(),
"show_flair": donor.show_flair,
}
user = lb_users.get(donor.editor_id)
if user:
donation["is_listenbrainz_user"] = True
donation["musicbrainz_id"] = user["musicbrainz_id"]
donation["flair"] = user["flair"]
else:
donation["is_listenbrainz_user"] = False
donation["musicbrainz_id"] = donor.editor_name
donation["flair"] = None

donors_with_flair.append(donation)

return donors_with_flair


def get_recent_donors(meb_conn, db_conn, count: int, offset: int):
""" Returns a list of recent donors with their flairs """
query = """
SELECT editor_name
, editor_id
, (amount + fee) as donation
, currency
, payment_date
-- check if the donation itself is eligible for flair
-- convert days to month because by default timestamp subtraction is in days
, bool_or(
(
(amount + fee)
/ ceiling(EXTRACT(days from now() - payment_date) / 30.0)
)
>= :threshold
) OVER (PARTITION BY editor_id) AS show_flair
FROM payment
WHERE editor_id IS NOT NULL
AND is_donation = 't'
AND payment_date >= (NOW() - INTERVAL '1 year')
ORDER BY payment_date DESC
LIMIT :count
OFFSET :offset
"""
results = meb_conn.execute(text(query), {
"count": count,
"offset": offset,
"threshold": FLAIR_MONTHLY_DONATION_THRESHOLD
})
donors = results.all()

return get_flairs_for_donors(db_conn, donors)


def get_biggest_donors(meb_conn, db_conn, count: int, offset: int):
""" Returns a list of biggest donors with their flairs """
query = """
WITH select_donations AS (
SELECT editor_name
, editor_id
, (amount + fee) as donation
, currency
, payment_date
-- check if the donation itself is eligible for flair
-- convert days to month because by default timestamp subtraction is in days
, (
(
(amount + fee)
/ ceiling(EXTRACT(days from now() - payment_date) / 30.0)
)
>= :threshold
) AS is_donation_eligible
FROM payment
WHERE editor_id IS NOT NULL
AND is_donation = 't'
AND payment_date >= (NOW() - INTERVAL '1 year')
)
SELECT editor_name
, editor_id
, currency
, max(payment_date) as payment_date
, sum(donation) as donation
, bool_or(is_donation_eligible) AS show_flair
FROM select_donations
GROUP BY editor_name
, editor_id
, currency
ORDER BY donation DESC
LIMIT :count
OFFSET :offset
"""

results = meb_conn.execute(text(query), {
"count": count,
"offset": offset,
"threshold": FLAIR_MONTHLY_DONATION_THRESHOLD
})
donors = results.all()

return get_flairs_for_donors(db_conn, donors)


def is_user_eligible_donor(meb_conn, musicbrainz_row_id: int):
""" Check if the user with the given musicbrainz row id is a donor and has enough recent
donations to be eligible for flair """
query = """
SELECT coalesce(
bool_or(
(
(amount + fee)
/ ceiling(EXTRACT(days from now() - payment_date) / 30.0)
)
>= :threshold)
, 'f'
) AS show_flair
FROM payment
WHERE editor_id = :editor_id
"""
result = meb_conn.execute(text(query), {
"editor_id": musicbrainz_row_id,
"threshold": FLAIR_MONTHLY_DONATION_THRESHOLD
})
row = result.first()
return row is not None and row.show_flair
25 changes: 25 additions & 0 deletions listenbrainz/db/user_setting.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import json

import sqlalchemy
from listenbrainz import db
from listenbrainz.db.exceptions import DatabaseException
Expand Down Expand Up @@ -136,3 +138,26 @@ def get_brainzplayer_prefs(db_conn, user_id: int):
"""), {"user_id": user_id})
row = result.mappings().first()
return dict(row) if row else None


def update_flair(db_conn, user_id: int, flair):
""" Update a user's flair """
db_conn.execute(sqlalchemy.text("""
INSERT INTO user_setting (user_id, flair)
VALUES (:user_id, :flair)
ON CONFLICT (user_id)
DO UPDATE
SET flair = EXCLUDED.flair
"""), {"flair": json.dumps(flair), "user_id": user_id})
db_conn.commit()


def get_flair(db_conn, user_id: int):
""" Retrieve flair for the given user """
result = db_conn.execute(sqlalchemy.text("""
SELECT flair
FROM user_setting
WHERE user_id = :user_id
"""), {"user_id": user_id})
row = result.mappings().first()
return row.flair if row else None
15 changes: 14 additions & 1 deletion listenbrainz/webserver/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from werkzeug.local import LocalProxy

from listenbrainz import db
from listenbrainz.db import create_test_database_connect_strings, timescale
from listenbrainz.db import create_test_database_connect_strings, timescale, donation
from listenbrainz.db.timescale import create_test_timescale_connect_strings

API_PREFIX = '/1'
Expand Down Expand Up @@ -39,8 +39,16 @@ def _get_ts_conn():
return _ts_conn


def _get_meb_conn():
_meb_conn = getattr(g, "_meb_conn", None)
if donation.engine is not None and _meb_conn is None:
_meb_conn = g._meb_conn = donation.engine.connect()
return _meb_conn


db_conn = LocalProxy(_get_db_conn)
ts_conn = LocalProxy(_get_ts_conn)
meb_conn = LocalProxy(_get_meb_conn)


def load_config(app):
Expand Down Expand Up @@ -123,6 +131,8 @@ def create_app(debug=None):
else:
db.init_db_connection(app.config["SQLALCHEMY_DATABASE_URI"])
timescale.init_db_connection(app.config["SQLALCHEMY_TIMESCALE_URI"])
if app.config.get("SQLALCHEMY_METABRAINZ_URI", None):
donation.init_meb_db_connection(app.config["SQLALCHEMY_METABRAINZ_URI"])

@app.teardown_request
def close_connection(exception):
Expand Down Expand Up @@ -402,6 +412,9 @@ def _register_blueprints(app):
from listenbrainz.webserver.views.popularity_api import popularity_api_bp
app.register_blueprint(popularity_api_bp, url_prefix=API_PREFIX+"/popularity")

from listenbrainz.webserver.views.donor_api import donor_api_bp
app.register_blueprint(donor_api_bp, url_prefix=API_PREFIX+"/donors")

from listenbrainz.webserver.views.entity_pages import artist_bp, album_bp, release_bp, release_group_bp
app.register_blueprint(artist_bp, url_prefix='/artist')
app.register_blueprint(album_bp, url_prefix='/album')
Expand Down
21 changes: 15 additions & 6 deletions listenbrainz/webserver/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@
from flask import current_app, request
from flask_login import current_user

from listenbrainz.webserver import db_conn
from listenbrainz.webserver import db_conn, meb_conn
from listenbrainz.webserver.views.views_utils import get_current_spotify_user, get_current_youtube_user, \
get_current_critiquebrainz_user, get_current_musicbrainz_user, get_current_soundcloud_user, get_current_apple_music_user
import listenbrainz.db.user_setting as db_usersetting
import listenbrainz.db.donation as db_donation

REJECT_LISTENS_WITHOUT_EMAIL_ERROR = \
'The listens were rejected because the user does not has not provided an email. ' \
Expand Down Expand Up @@ -77,11 +78,19 @@ def get_global_props():
"sentry_traces_sample_rate": sentry_config.get("traces_sample_rate", 0.0),
}

brainzplayer_props = db_usersetting.get_brainzplayer_prefs(
db_conn, current_user.id
) if current_user.is_authenticated else None
if brainzplayer_props is not None:
props["user_preferences"] = brainzplayer_props
if current_user.is_authenticated:
brainzplayer_props = db_usersetting.get_brainzplayer_prefs(db_conn, current_user.id)
if brainzplayer_props is not None:
props["user_preferences"] = brainzplayer_props

flair_props = db_usersetting.get_flair(db_conn, current_user.id)
if flair_props is not None:
props["flair"] = flair_props

if meb_conn:
show_flair = db_donation.is_user_eligible_donor(meb_conn, current_user.id)
if show_flair is not None:
props["show_flair"] = show_flair

return orjson.dumps(props).decode("utf-8")

Expand Down
40 changes: 40 additions & 0 deletions listenbrainz/webserver/views/donor_api.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
from brainzutils.ratelimit import ratelimit
from flask import Blueprint, jsonify

from listenbrainz.db.donation import get_recent_donors, get_biggest_donors
from listenbrainz.webserver import db_conn, meb_conn
from listenbrainz.webserver.decorators import crossdomain
from listenbrainz.webserver.views.api_tools import _parse_int_arg

donor_api_bp = Blueprint('donor_api_v1', __name__)


DEFAULT_DONOR_COUNT = 25


@donor_api_bp.route("/recent", methods=["GET", "OPTIONS"])
@crossdomain
@ratelimit()
def recent_donors():
""" Get a list of most recent MeB donors with flairs.

"""
count = _parse_int_arg("count", DEFAULT_DONOR_COUNT)
offset = _parse_int_arg("offset", 0)

donors = get_recent_donors(meb_conn, db_conn, count, offset)
return jsonify(donors)


@donor_api_bp.route("/biggest", methods=["GET", "OPTIONS"])
@crossdomain
@ratelimit()
def biggest_donors():
""" Get a list of the biggest MeB donors with flairs.

"""
count = _parse_int_arg("count", DEFAULT_DONOR_COUNT)
offset = _parse_int_arg("offset", 0)

donors = get_biggest_donors(meb_conn, db_conn, count, offset)
return jsonify(donors)
Loading