From 8b9a7893d3fd03fb2c8c1396f33372ea945f2993 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 19 Jul 2024 12:10:51 -0400 Subject: [PATCH 01/23] feat(mpd): Implement MPD Source MVP --- README.md | 1 + config/mpd.json.example | 13 ++ docsite/docs/configuration/configuration.mdx | 47 ++++ docsite/src/pages/index.mdx | 1 + package-lock.json | 102 +++++++++ package.json | 1 + src/backend/common/infrastructure/Atomic.ts | 6 +- .../infrastructure/config/source/mpd.ts | 70 ++++++ .../infrastructure/config/source/sources.ts | 7 +- src/backend/common/schema/aio-source.json | 166 ++++++++++++++ src/backend/common/schema/aio.json | 166 ++++++++++++++ src/backend/common/schema/source.json | 158 +++++++++++++ src/backend/sources/MPDSource.ts | 216 ++++++++++++++++++ src/backend/sources/ScrobbleSources.ts | 5 + src/backend/utils/NetworkUtils.ts | 41 ++++ src/core/Atomic.ts | 16 +- 16 files changed, 1005 insertions(+), 11 deletions(-) create mode 100644 config/mpd.json.example create mode 100644 src/backend/common/infrastructure/config/source/mpd.ts create mode 100644 src/backend/sources/MPDSource.ts create mode 100644 src/backend/utils/NetworkUtils.ts diff --git a/README.md b/README.md index affa2cb0..761b1b50 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * [Kodi](https://foxxmd.github.io/multi-scrobbler/docs/configuration#kodi) * [Google Cast (Chromecast)](https://foxxmd.github.io/multi-scrobbler/docs/configuration#google-cast-chromecast) * [Musikcube](https://foxxmd.github.io/multi-scrobbler/docs/configuration#muikcube) + * [MPD (Music Player Daemon)](https://foxxmd.github.io/multi-scrobbler/docs/configuration#mpd-music-player-daemon) * Supports scrobbling to many **Clients** * [Maloja](https://foxxmd.github.io/multi-scrobbler/docs/configuration#maloja) * [Last.fm](https://foxxmd.github.io/multi-scrobbler/docs/configuration#lastfm) diff --git a/config/mpd.json.example b/config/mpd.json.example new file mode 100644 index 00000000..fc4d5815 --- /dev/null +++ b/config/mpd.json.example @@ -0,0 +1,13 @@ +[ + { + "enable": true, + "name": "MyMPD", + "data": { + "url": "192.168.0.100:6600", + "password": "MY_PASSWORD" + }, + "options": { + "disableDiscovery": false + } + } +] diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index 6b0c050c..7b9d3026 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -21,6 +21,7 @@ import MalojaConfig from '!!raw-loader!../../../config/maloja.json.example'; import MopidyConfig from '!!raw-loader!../../../config/mopidy.json.example'; import MprisConfig from '!!raw-loader!../../../config/mpris.json.example'; import MusikcubeConfig from '!!raw-loader!../../../config/musikcube.json.example'; +import MPDConfig from '!!raw-loader!../../../config/mpd.json.example'; import PlexConfig from '!!raw-loader!../../../config/plex.json.example'; import SpotifyConfig from '!!raw-loader!../../../config/spotify.json.example'; import SubsonicConfig from '!!raw-loader!../../../config/subsonic.json.example'; @@ -1256,6 +1257,52 @@ If no URL is provided to MS it will try to use `ws://localhost:7905` +### [MPD (Music Player Daemon)](https://www.musicpd.org/) + +MS communicates with MPD using the [TCP client connection.](https://mpd.readthedocs.io/en/stable/user.html#client-connections) + +You should uncomment/create the following settings in your mpd config: + +``` +bind_to_address "any" # or a specific ipv4/v6 address +port "6600" +``` + + + +#### Configuration + + + + | Environmental Variable | Required? | Default | Description | + |------------------------|-----------|------------------|-------------| + | `MPD_URL` | No | `localhost:6600` | | + | `MPD_PASSWORD` | No | | | + + +
+ + Example + + {MPDConfig} + +
+ + or + + +
+ + Example + + + +
+ + or + + + ## Client Configurations ### [Maloja](https://github.com/krateng/maloja) diff --git a/docsite/src/pages/index.mdx b/docsite/src/pages/index.mdx index be25f971..c5ff95e8 100644 --- a/docsite/src/pages/index.mdx +++ b/docsite/src/pages/index.mdx @@ -28,6 +28,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * [Kodi](docs/configuration#kodi) * [Google Cast (Chromecast)](docs/configuration#google-cast-chromecast) * [Musikcube](docs/configuration#musikcube) + * [MPD (Music Player Daemon)](docs/configuration#mpd-music-player-daemon) * Supports scrobbling to many **Clients** * [Maloja](docs/configuration#maloja) * [Last.fm](docs/configuration#lastfm) diff --git a/package-lock.json b/package-lock.json index 17d59509..3dc6364d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,6 +51,7 @@ "kodi-api": "^0.2.1", "lastfm-node-client": "^2.2.0", "mopidy": "^1.3.0", + "mpd-api": "^1.1.2", "nanoid": "^3.3.1", "normalize-url": "^8.0.1", "ntfy": "^1.5.4", @@ -2150,6 +2151,11 @@ "@testing-library/dom": ">=7.21.4" } }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==" + }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -4852,6 +4858,22 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-type": { + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "dependencies": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -6934,6 +6956,27 @@ } } }, + "node_modules/mpd-api": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/mpd-api/-/mpd-api-1.1.2.tgz", + "integrity": "sha512-UWSkIzYQnTvuvhLTfD0bIhBa4PCGCz6OsnDlojp319khsL8xXwS4qLVyJ+R5J7gNUTWiV9Y6d1j6apSYt2WDVA==", + "dependencies": { + "debug": "^4.3.4", + "file-type": "^16.5.3", + "mpd2": "^1.0.5" + }, + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/mpd2": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/mpd2/-/mpd2-1.0.5.tgz", + "integrity": "sha512-DcIy3jISfjmFTlOpwDE2KDhBLcGuFIAMTvY23OMkCikJXRnDqqm1jaHG3/LVYkKjrUVzQhlABW31roGqv/esRw==", + "dependencies": { + "debug": "^4.1.1" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -7927,6 +7970,18 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/picocolors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", @@ -8633,6 +8688,21 @@ "node": ">= 6" } }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "dependencies": { + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -9426,6 +9496,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/sucrase": { "version": "3.35.0", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", @@ -9744,6 +9830,22 @@ "node": ">=0.6" } }, + "node_modules/token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, "node_modules/touch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", diff --git a/package.json b/package.json index 4a161d5f..1178db61 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "kodi-api": "^0.2.1", "lastfm-node-client": "^2.2.0", "mopidy": "^1.3.0", + "mpd-api": "^1.1.2", "nanoid": "^3.3.1", "normalize-url": "^8.0.1", "ntfy": "^1.5.4", diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 6cdf80cc..897aa747 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -22,7 +22,8 @@ export type SourceType = | 'kodi' | 'webscrobbler' | 'chromecast' - | 'musikcube'; + | 'musikcube' + | 'mpd'; export const sourceTypes: SourceType[] = [ 'spotify', @@ -40,7 +41,8 @@ export const sourceTypes: SourceType[] = [ 'kodi', 'webscrobbler', 'chromecast', - 'musikcube' + 'musikcube', + 'mpd' ]; export const lowGranularitySources: SourceType[] = ['subsonic', 'ytmusic']; diff --git a/src/backend/common/infrastructure/config/source/mpd.ts b/src/backend/common/infrastructure/config/source/mpd.ts new file mode 100644 index 00000000..57297d07 --- /dev/null +++ b/src/backend/common/infrastructure/config/source/mpd.ts @@ -0,0 +1,70 @@ +import { CommonSourceConfig, CommonSourceData, CommonSourceOptions } from "./index.js"; + +export interface MPDData extends CommonSourceData { + /** + * URL:PORT of the MPD server to connect to + * + * To use this you must have TCP connections enabled for your MPD server https://mpd.readthedocs.io/en/stable/user.html#client-connections + * + * @examples ["localhost:6600"] + * @default "localhost:6600" + * */ + url?: string + + /** + * If using socket specify the path instead of url. + * + * trailing `~` is replaced by your home directory + * */ + path?: string + + /** + * Password for the server, if set https://mpd.readthedocs.io/en/stable/user.html#permissions-and-passwords + * */ + password?: string + +} + +export interface MPDSourceOptions extends CommonSourceOptions { + //disableDiscovery?: boolean +} + +export interface MPDSourceConfig extends CommonSourceConfig { + data: MPDData + options: MPDSourceOptions +} + +export interface MPDSourceAIOConfig extends MPDSourceConfig { + type: 'mpd' +} + +export type PlayerState = 'play' | 'stop' | 'pause'; + +export interface StatusResponse { + state: PlayerState + /** + * Position within the current song in seconds + * */ + elapsed?: number + /** + * Duration of the current song in seconds + * */ + duration?: number + error?: string +} + +export interface CurrentSongResponse { + file: string + time: number + name?: string + performer?: string + artist?: string + album?: string + albumartist?: string + title?: string + musicbrainz_albumartistid?: string + musicbrainz_albumid?: string + musicbrainz_artistid?: string + musicbrainz_releasetrackid?: string + musicbrainz_trackid?: string +} diff --git a/src/backend/common/infrastructure/config/source/sources.ts b/src/backend/common/infrastructure/config/source/sources.ts index 6a42ed80..48f95e08 100644 --- a/src/backend/common/infrastructure/config/source/sources.ts +++ b/src/backend/common/infrastructure/config/source/sources.ts @@ -6,6 +6,7 @@ import { KodiSourceAIOConfig, KodiSourceConfig } from "./kodi.js"; import { LastFmSouceAIOConfig, LastfmSourceConfig } from "./lastfm.js"; import { ListenBrainzSourceAIOConfig, ListenBrainzSourceConfig } from "./listenbrainz.js"; import { MopidySourceAIOConfig, MopidySourceConfig } from "./mopidy.js"; +import { MPDSourceAIOConfig, MPDSourceConfig } from "./mpd.js"; import { MPRISSourceAIOConfig, MPRISSourceConfig } from "./mpris.js"; import { MusikcubeSourceAIOConfig, MusikcubeSourceConfig } from "./musikcube.js"; import { PlexSourceAIOConfig, PlexSourceConfig } from "./plex.js"; @@ -32,7 +33,8 @@ export type SourceConfig = | KodiSourceConfig | WebScrobblerSourceConfig | ChromecastSourceConfig - | MusikcubeSourceConfig; + | MusikcubeSourceConfig + | MPDSourceConfig; export type SourceAIOConfig = SpotifySourceAIOConfig @@ -50,4 +52,5 @@ export type SourceAIOConfig = | KodiSourceAIOConfig | WebScrobblerSourceAIOConfig | ChromecastSourceAIOConfig - | MusikcubeSourceAIOConfig; + | MusikcubeSourceAIOConfig + | MPDSourceAIOConfig; diff --git a/src/backend/common/schema/aio-source.json b/src/backend/common/schema/aio-source.json index b5d3ae91..4ec9ab3c 100644 --- a/src/backend/common/schema/aio-source.json +++ b/src/backend/common/schema/aio-source.json @@ -969,6 +969,169 @@ "title": "ListenBrainzSourceData", "type": "object" }, + "MPDData": { + "properties": { + "password": { + "description": "Password for the server, if set https://mpd.readthedocs.io/en/stable/user.html#permissions-and-passwords", + "title": "password", + "type": "string" + }, + "path": { + "description": "If using socket specify the path instead of url.\n\ntrailing `~` is replaced by your home directory", + "title": "path", + "type": "string" + }, + "url": { + "default": "localhost:6600", + "description": "URL:PORT of the MPD server to connect to\n\nTo use this you must have TCP connections enabled for your MPD server https://mpd.readthedocs.io/en/stable/user.html#client-connections", + "examples": [ + "localhost:6600" + ], + "title": "url", + "type": "string" + } + }, + "title": "MPDData", + "type": "object" + }, + "MPDSourceAIOConfig": { + "properties": { + "clients": { + "description": "Restrict scrobbling tracks played from this source to Clients with names from this list. If list is empty is not present Source scrobbles to all configured Clients.", + "examples": [ + [ + "MyMalojaConfigName", + "MyLastFMConfigName" + ] + ], + "items": { + "type": "string" + }, + "title": "clients", + "type": "array" + }, + "data": { + "$ref": "#/definitions/MPDData", + "title": "data" + }, + "enable": { + "default": true, + "description": "Should MS use this client/source? Defaults to true", + "examples": [ + true + ], + "title": "enable", + "type": "boolean" + }, + "name": { + "description": "Unique identifier for this source.", + "title": "name", + "type": "string" + }, + "options": { + "$ref": "#/definitions/MPDSourceOptions", + "title": "options" + }, + "type": { + "enum": [ + "mpd" + ], + "title": "type", + "type": "string" + } + }, + "required": [ + "data", + "options", + "type" + ], + "title": "MPDSourceAIOConfig", + "type": "object" + }, + "MPDSourceOptions": { + "properties": { + "logFilterFailure": { + "default": "warn", + "description": "If this source has INGRESS to MS and has filters this determines how MS logs when a payload (event) fails a defined filter (IE users/servers/library filters)\n\n* `false` => do not log\n* `debug` => log to DEBUG level\n* `warn` => log to WARN level (default)\n\nHint: This is useful if you are sure this source is setup correctly and you have multiple other sources. Set to `debug` or `false` to reduce log noise.", + "enum": [ + "debug", + false, + "warn" + ], + "examples": [ + "warn" + ], + "title": "logFilterFailure" + }, + "logPayload": { + "default": false, + "description": "If this source has INGRESS to MS (sends a payload, rather than MS GETTING requesting a payload)\nthen setting this option to true will make MS log the payload JSON to DEBUG output", + "examples": [ + false + ], + "title": "logPayload", + "type": "boolean" + }, + "logPlayerState": { + "default": false, + "description": "For Sources that track Player State (currently playing) this logs a simple player state/summary to DEBUG output", + "examples": [ + false + ], + "title": "logPlayerState", + "type": "boolean" + }, + "maxPollRetries": { + "default": 5, + "description": "default # of automatic polling restarts on error", + "examples": [ + 5 + ], + "title": "maxPollRetries", + "type": "number" + }, + "maxRequestRetries": { + "default": 1, + "description": "default # of http request retries a source/client can make before error is thrown", + "examples": [ + 1 + ], + "title": "maxRequestRetries", + "type": "number" + }, + "retryMultiplier": { + "default": 1.5, + "description": "default retry delay multiplier (retry attempt * multiplier = # of seconds to wait before retrying)", + "examples": [ + 1.5 + ], + "title": "retryMultiplier", + "type": "number" + }, + "scrobbleBacklog": { + "default": true, + "description": "If this source\n\n* supports fetching a listen history\n* and this option is enabled\n\nthen on startup MS will attempt to scrobble the recent listens from that history", + "examples": [ + true, + false + ], + "title": "scrobbleBacklog", + "type": "boolean" + }, + "scrobbleBacklogCount": { + "description": "The number of listens to fetch when scrobbling from backlog\n\n* Only applies if this source supports fetching a listen history\n* If not specified it defaults to the maximum number of listens the source API supports", + "title": "scrobbleBacklogCount", + "type": "number" + }, + "scrobbleThresholds": { + "$ref": "#/definitions/ScrobbleThresholds", + "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.", + "title": "scrobbleThresholds" + } + }, + "title": "MPDSourceOptions", + "type": "object" + }, "MPRISData": { "properties": { "blacklist": { @@ -1440,6 +1603,9 @@ { "$ref": "#/definitions/MopidySourceAIOConfig" }, + { + "$ref": "#/definitions/MPDSourceAIOConfig" + }, { "$ref": "#/definitions/MPRISSourceAIOConfig" }, diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json index 55a4560a..6e62b9b8 100644 --- a/src/backend/common/schema/aio.json +++ b/src/backend/common/schema/aio.json @@ -1603,6 +1603,169 @@ "title": "LogOptions", "type": "object" }, + "MPDData": { + "properties": { + "password": { + "description": "Password for the server, if set https://mpd.readthedocs.io/en/stable/user.html#permissions-and-passwords", + "title": "password", + "type": "string" + }, + "path": { + "description": "If using socket specify the path instead of url.\n\ntrailing `~` is replaced by your home directory", + "title": "path", + "type": "string" + }, + "url": { + "default": "localhost:6600", + "description": "URL:PORT of the MPD server to connect to\n\nTo use this you must have TCP connections enabled for your MPD server https://mpd.readthedocs.io/en/stable/user.html#client-connections", + "examples": [ + "localhost:6600" + ], + "title": "url", + "type": "string" + } + }, + "title": "MPDData", + "type": "object" + }, + "MPDSourceAIOConfig": { + "properties": { + "clients": { + "description": "Restrict scrobbling tracks played from this source to Clients with names from this list. If list is empty is not present Source scrobbles to all configured Clients.", + "examples": [ + [ + "MyMalojaConfigName", + "MyLastFMConfigName" + ] + ], + "items": { + "type": "string" + }, + "title": "clients", + "type": "array" + }, + "data": { + "$ref": "#/definitions/MPDData", + "title": "data" + }, + "enable": { + "default": true, + "description": "Should MS use this client/source? Defaults to true", + "examples": [ + true + ], + "title": "enable", + "type": "boolean" + }, + "name": { + "description": "Unique identifier for this source.", + "title": "name", + "type": "string" + }, + "options": { + "$ref": "#/definitions/MPDSourceOptions", + "title": "options" + }, + "type": { + "enum": [ + "mpd" + ], + "title": "type", + "type": "string" + } + }, + "required": [ + "data", + "options", + "type" + ], + "title": "MPDSourceAIOConfig", + "type": "object" + }, + "MPDSourceOptions": { + "properties": { + "logFilterFailure": { + "default": "warn", + "description": "If this source has INGRESS to MS and has filters this determines how MS logs when a payload (event) fails a defined filter (IE users/servers/library filters)\n\n* `false` => do not log\n* `debug` => log to DEBUG level\n* `warn` => log to WARN level (default)\n\nHint: This is useful if you are sure this source is setup correctly and you have multiple other sources. Set to `debug` or `false` to reduce log noise.", + "enum": [ + "debug", + false, + "warn" + ], + "examples": [ + "warn" + ], + "title": "logFilterFailure" + }, + "logPayload": { + "default": false, + "description": "If this source has INGRESS to MS (sends a payload, rather than MS GETTING requesting a payload)\nthen setting this option to true will make MS log the payload JSON to DEBUG output", + "examples": [ + false + ], + "title": "logPayload", + "type": "boolean" + }, + "logPlayerState": { + "default": false, + "description": "For Sources that track Player State (currently playing) this logs a simple player state/summary to DEBUG output", + "examples": [ + false + ], + "title": "logPlayerState", + "type": "boolean" + }, + "maxPollRetries": { + "default": 5, + "description": "default # of automatic polling restarts on error", + "examples": [ + 5 + ], + "title": "maxPollRetries", + "type": "number" + }, + "maxRequestRetries": { + "default": 1, + "description": "default # of http request retries a source/client can make before error is thrown", + "examples": [ + 1 + ], + "title": "maxRequestRetries", + "type": "number" + }, + "retryMultiplier": { + "default": 1.5, + "description": "default retry delay multiplier (retry attempt * multiplier = # of seconds to wait before retrying)", + "examples": [ + 1.5 + ], + "title": "retryMultiplier", + "type": "number" + }, + "scrobbleBacklog": { + "default": true, + "description": "If this source\n\n* supports fetching a listen history\n* and this option is enabled\n\nthen on startup MS will attempt to scrobble the recent listens from that history", + "examples": [ + true, + false + ], + "title": "scrobbleBacklog", + "type": "boolean" + }, + "scrobbleBacklogCount": { + "description": "The number of listens to fetch when scrobbling from backlog\n\n* Only applies if this source supports fetching a listen history\n* If not specified it defaults to the maximum number of listens the source API supports", + "title": "scrobbleBacklogCount", + "type": "number" + }, + "scrobbleThresholds": { + "$ref": "#/definitions/ScrobbleThresholds", + "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.", + "title": "scrobbleThresholds" + } + }, + "title": "MPDSourceOptions", + "type": "object" + }, "MPRISData": { "properties": { "blacklist": { @@ -2285,6 +2448,9 @@ { "$ref": "#/definitions/MopidySourceAIOConfig" }, + { + "$ref": "#/definitions/MPDSourceAIOConfig" + }, { "$ref": "#/definitions/MPRISSourceAIOConfig" }, diff --git a/src/backend/common/schema/source.json b/src/backend/common/schema/source.json index 8d15a29c..0e28b0ff 100644 --- a/src/backend/common/schema/source.json +++ b/src/backend/common/schema/source.json @@ -25,6 +25,9 @@ { "$ref": "#/definitions/MopidySourceConfig" }, + { + "$ref": "#/definitions/MPDSourceConfig" + }, { "$ref": "#/definitions/MPRISSourceConfig" }, @@ -963,6 +966,161 @@ "title": "ListenBrainzSourceData", "type": "object" }, + "MPDData": { + "properties": { + "password": { + "description": "Password for the server, if set https://mpd.readthedocs.io/en/stable/user.html#permissions-and-passwords", + "title": "password", + "type": "string" + }, + "path": { + "description": "If using socket specify the path instead of url.\n\ntrailing `~` is replaced by your home directory", + "title": "path", + "type": "string" + }, + "url": { + "default": "localhost:6600", + "description": "URL:PORT of the MPD server to connect to\n\nTo use this you must have TCP connections enabled for your MPD server https://mpd.readthedocs.io/en/stable/user.html#client-connections", + "examples": [ + "localhost:6600" + ], + "title": "url", + "type": "string" + } + }, + "title": "MPDData", + "type": "object" + }, + "MPDSourceConfig": { + "properties": { + "clients": { + "description": "Restrict scrobbling tracks played from this source to Clients with names from this list. If list is empty is not present Source scrobbles to all configured Clients.", + "examples": [ + [ + "MyMalojaConfigName", + "MyLastFMConfigName" + ] + ], + "items": { + "type": "string" + }, + "title": "clients", + "type": "array" + }, + "data": { + "$ref": "#/definitions/MPDData", + "title": "data" + }, + "enable": { + "default": true, + "description": "Should MS use this client/source? Defaults to true", + "examples": [ + true + ], + "title": "enable", + "type": "boolean" + }, + "name": { + "description": "Unique identifier for this source.", + "title": "name", + "type": "string" + }, + "options": { + "$ref": "#/definitions/MPDSourceOptions", + "title": "options" + } + }, + "required": [ + "data", + "options" + ], + "title": "MPDSourceConfig", + "type": "object" + }, + "MPDSourceOptions": { + "properties": { + "logFilterFailure": { + "default": "warn", + "description": "If this source has INGRESS to MS and has filters this determines how MS logs when a payload (event) fails a defined filter (IE users/servers/library filters)\n\n* `false` => do not log\n* `debug` => log to DEBUG level\n* `warn` => log to WARN level (default)\n\nHint: This is useful if you are sure this source is setup correctly and you have multiple other sources. Set to `debug` or `false` to reduce log noise.", + "enum": [ + "debug", + false, + "warn" + ], + "examples": [ + "warn" + ], + "title": "logFilterFailure" + }, + "logPayload": { + "default": false, + "description": "If this source has INGRESS to MS (sends a payload, rather than MS GETTING requesting a payload)\nthen setting this option to true will make MS log the payload JSON to DEBUG output", + "examples": [ + false + ], + "title": "logPayload", + "type": "boolean" + }, + "logPlayerState": { + "default": false, + "description": "For Sources that track Player State (currently playing) this logs a simple player state/summary to DEBUG output", + "examples": [ + false + ], + "title": "logPlayerState", + "type": "boolean" + }, + "maxPollRetries": { + "default": 5, + "description": "default # of automatic polling restarts on error", + "examples": [ + 5 + ], + "title": "maxPollRetries", + "type": "number" + }, + "maxRequestRetries": { + "default": 1, + "description": "default # of http request retries a source/client can make before error is thrown", + "examples": [ + 1 + ], + "title": "maxRequestRetries", + "type": "number" + }, + "retryMultiplier": { + "default": 1.5, + "description": "default retry delay multiplier (retry attempt * multiplier = # of seconds to wait before retrying)", + "examples": [ + 1.5 + ], + "title": "retryMultiplier", + "type": "number" + }, + "scrobbleBacklog": { + "default": true, + "description": "If this source\n\n* supports fetching a listen history\n* and this option is enabled\n\nthen on startup MS will attempt to scrobble the recent listens from that history", + "examples": [ + true, + false + ], + "title": "scrobbleBacklog", + "type": "boolean" + }, + "scrobbleBacklogCount": { + "description": "The number of listens to fetch when scrobbling from backlog\n\n* Only applies if this source supports fetching a listen history\n* If not specified it defaults to the maximum number of listens the source API supports", + "title": "scrobbleBacklogCount", + "type": "number" + }, + "scrobbleThresholds": { + "$ref": "#/definitions/ScrobbleThresholds", + "description": "Set thresholds for when multi-scrobbler should consider a tracked play to be \"scrobbable\". If both duration and percent are defined then if either condition is met the track is scrobbled.", + "title": "scrobbleThresholds" + } + }, + "title": "MPDSourceOptions", + "type": "object" + }, "MPRISData": { "properties": { "blacklist": { diff --git a/src/backend/sources/MPDSource.ts b/src/backend/sources/MPDSource.ts new file mode 100644 index 00000000..8049db2f --- /dev/null +++ b/src/backend/sources/MPDSource.ts @@ -0,0 +1,216 @@ +import { EventEmitter } from "events"; +import mpdapiNS, { MPDApi } from 'mpd-api'; +import mpd2 from 'mpd2'; +import { BrainzMeta, PlayObject } from "../../core/Atomic.js"; +import { + FormatPlayObjectOptions, + InternalConfig, + PlayerStateData, + REPORTED_PLAYER_STATUSES, + ReportedPlayerStatus, + SINGLE_USER_PLATFORM_ID, +} from "../common/infrastructure/Atomic.js"; +import { + CurrentSongResponse, + MPDSourceConfig, + PlayerState, + StatusResponse, +} from "../common/infrastructure/config/source/mpd.js"; +import { isPortReachable } from "../utils/NetworkUtils.js"; +import { RecentlyPlayedOptions } from "./AbstractSource.js"; +import MemorySource from "./MemorySource.js"; + +const mpdClient = mpdapiNS.default; + +const CLIENT_PLAYER_STATE: Record = { + 'play': REPORTED_PLAYER_STATUSES.playing, + 'pause': REPORTED_PLAYER_STATUSES.paused, + 'stop': REPORTED_PLAYER_STATUSES.stopped, +} + +export class MPDSource extends MemorySource { + declare config: MPDSourceConfig; + + host?: string + port?: number + // {host?: string, port?: number, path?: string, password?: string}; + clientConfig: mpd2.MPD.Config; + client!: MPDApi.ClientAPI; + deviceId: string + + constructor(name: any, config: MPDSourceConfig, internal: InternalConfig, emitter: EventEmitter) { + const { + data = {} + } = config; + const { + ...rest + } = data; + super('mpd', name, {...config, data: {...rest}}, internal, emitter); + + this.requiresAuth = true; + this.canPoll = true; + } + + static parseConnectionUrl(valRaw: string): [string, string] { + if(valRaw.trim() === '') { + throw new Error(`'url' cannot be an empty string`); + } + + const [host, port] = valRaw.trim().split(':'); + return [host, port ?? '6600']; + } + + protected async doBuildInitData(): Promise { + const { + data: { + url, + path, + password, + } = {} + } = this.config; + + if(path === undefined) { + const [host, port] = MPDSource.parseConnectionUrl(url ?? 'localhost:6600'); + this.logger.verbose(`Config URL: '${url ?? '(None Given)'}' => Normalized: '${host}:${port}'`); + this.host = host; + this.port = Number.parseInt(port); + this.clientConfig = { + host, + port: this.port, + password + } + } else { + this.logger.verbose(`Using socket path: ${path}`); + this.clientConfig = { + path, + password + } + } + + return true; + } + + protected async doCheckConnection(): Promise { + if(this.host !== undefined) { + try { + await isPortReachable(this.port, {host: this.host}); + return `${this.host}:${this.port} is reachable.`; + } catch (e) { + throw e; + } + } + return null; + } + + doAuthentication = async () => { + + try { + this.client = await mpdClient.connect({...this.clientConfig, timeout: 1000}); + return true; + } catch (e) { + let friendlyError: string | undefined; + if(e.code === 'ENOENT') { + friendlyError = 'Socket file does not exist' + } else if(e.code === 'EACCES') { + friendlyError = 'Incorrect permissions to access socket file' + } + // if(e.errno !== undefined) { + // switch(e.errno) { + // case mpd2.default.MPDError.CODES.PERMISSION: + // friendlyError = 'No permission to connect'; + // break; + // case mpd2.default.MPDError.CODES.PASSWORD: + // friendlyError = 'Password is probably not correct'; + // break; + // } + // } + throw new Error(`Could not connect to MPD server${friendlyError !== undefined ? ` (Hint: ${friendlyError})` : ''}`, {cause: e}); + } + } + + formatPlayObj(obj: CurrentSongResponse, options: FormatPlayObjectOptions = {}): PlayObject { + const { + file, + time, + artist, + performer, + album, + albumartist, + title, + name, + musicbrainz_albumartistid, + musicbrainz_albumid, + musicbrainz_artistid, + musicbrainz_releasetrackid, + musicbrainz_trackid, + } = obj; + + let artists = []; + let albumArtists = []; + if(artist !== undefined) { + artists.push(artist); + } + if(albumartist !== undefined && albumartist !== artist) { + albumArtists.push(albumartist); + } + if(artists.length === 0 && performer !== undefined) { + artists.push(performer); + } + if(artists.length === 0 && albumArtists.length !== 0) { + // switch these, tags are probably improper + artists = albumArtists; + albumArtists = []; + } + + let trackName = title; + if(trackName === undefined && name !== undefined) { + trackName = name; + } else if(trackName === undefined && file !== undefined) { + trackName = file; + } + + const brainz: BrainzMeta = { + albumArtist: musicbrainz_albumartistid, + album: musicbrainz_albumid, + track: musicbrainz_trackid, + }; + if(musicbrainz_artistid !== undefined) { + brainz.artist = [musicbrainz_artistid]; + } + + return { + data: { + artists: artists, + albumArtists, + album, + track: trackName, + duration: time + }, + meta: { + brainz, + trackProgressPosition: options.trackProgressPosition, + } + } + } + + getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => { + + const state = await this.client.api.status.get(); + const currentSong = await this.client.api.status.currentsong(); + + let play: PlayObject | undefined; + if(currentSong !== undefined) { + play = this.formatPlayObj(currentSong, {trackProgressPosition: state.elapsed}); + } + + const playerState: PlayerStateData = { + platformId: SINGLE_USER_PLATFORM_ID, + status: CLIENT_PLAYER_STATE[state.state], + play, + position: state.elapsed + } + + return this.processRecentPlays([playerState]); + } + +} diff --git a/src/backend/sources/ScrobbleSources.ts b/src/backend/sources/ScrobbleSources.ts index ccfee5e4..3cd6e46a 100644 --- a/src/backend/sources/ScrobbleSources.ts +++ b/src/backend/sources/ScrobbleSources.ts @@ -12,6 +12,7 @@ import { KodiData, KodiSourceConfig } from "../common/infrastructure/config/sour import { LastfmSourceConfig } from "../common/infrastructure/config/source/lastfm.js"; import { ListenBrainzSourceConfig } from "../common/infrastructure/config/source/listenbrainz.js"; import { MopidySourceConfig } from "../common/infrastructure/config/source/mopidy.js"; +import { MPDSourceConfig } from "../common/infrastructure/config/source/mpd.js"; import { MPRISData, MPRISSourceConfig } from "../common/infrastructure/config/source/mpris.js"; import { MusikcubeData, MusikcubeSourceConfig } from "../common/infrastructure/config/source/musikcube.js"; import { PlexSourceConfig } from "../common/infrastructure/config/source/plex.js"; @@ -34,6 +35,7 @@ import { KodiSource } from "./KodiSource.js"; import LastfmSource from "./LastfmSource.js"; import ListenbrainzSource from "./ListenbrainzSource.js"; import { MopidySource } from "./MopidySource.js"; +import { MPDSource } from "./MPDSource.js"; import { MPRISSource } from "./MPRISSource.js"; import { MusikcubeSource } from "./MusikcubeSource.js"; import PlexSource from "./PlexSource.js"; @@ -543,6 +545,9 @@ export default class ScrobbleSources { case 'musikcube': newSource = await new MusikcubeSource(name, compositeConfig as MusikcubeSourceConfig, internal, this.emitter); break; + case 'mpd': + newSource = await new MPDSource(name, compositeConfig as MPDSourceConfig, internal, this.emitter); + break; default: break; } diff --git a/src/backend/utils/NetworkUtils.ts b/src/backend/utils/NetworkUtils.ts new file mode 100644 index 00000000..29684780 --- /dev/null +++ b/src/backend/utils/NetworkUtils.ts @@ -0,0 +1,41 @@ +import net from 'node:net'; + +export interface PortReachableOpts { + host: string, + timeout?: number +} +/** + * Copied from https://github.com/sindresorhus/is-port-reachable with error reporting + * */ +export const isPortReachable = async (port: number, opts: PortReachableOpts) => { + const {host, timeout = 1000} = opts; + + const promise = new Promise(((resolve, reject) => { + const socket = new net.Socket(); + + const onError = (e) => { + socket.destroy(); + reject(e); + }; + const onTimeout = () => { + socket.destroy(); + reject(new Error(`Connection timed out after ${timeout}ms`)); + } + + socket.setTimeout(timeout); + socket.once('error', onError); + socket.once('timeout', onTimeout); + + socket.connect(port, host, () => { + socket.end(); + resolve(true); + }); + })); + + try { + await promise; + return true; + } catch (e) { + throw e; + } +} diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index cba13876..363de34a 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -57,6 +57,14 @@ export interface ListenRangeData { end: ListenProgress } +export interface BrainzMeta { + artist?: string[] + albumArtist?: string + album?: string + track?: string + releaseGroup?: string +} + export interface TrackData { artists?: string[] albumArtists?: string[] @@ -68,13 +76,7 @@ export interface TrackData { duration?: number meta?: { - brainz?: { - artist?: string[] - albumArtist?: string - album?: string - track?: string - releaseGroup?: string - } + brainz?: BrainzMeta } } From 7760776a5fcedc50237b9128b24defa2ed648388 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 19 Jul 2024 14:36:27 -0400 Subject: [PATCH 02/23] feat(mpd): Use idle event to wake up from polling early --- src/backend/sources/MPDSource.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/backend/sources/MPDSource.ts b/src/backend/sources/MPDSource.ts index 8049db2f..9329c5fb 100644 --- a/src/backend/sources/MPDSource.ts +++ b/src/backend/sources/MPDSource.ts @@ -1,3 +1,4 @@ +import dayjs from "dayjs"; import { EventEmitter } from "events"; import mpdapiNS, { MPDApi } from 'mpd-api'; import mpd2 from 'mpd2'; @@ -43,9 +44,10 @@ export class MPDSource extends MemorySource { data = {} } = config; const { + interval = 5, // reduced polling interval because its likely we are on the same network ...rest } = data; - super('mpd', name, {...config, data: {...rest}}, internal, emitter); + super('mpd', name, {...config, data: {...rest, interval}}, internal, emitter); this.requiresAuth = true; this.canPoll = true; @@ -106,6 +108,13 @@ export class MPDSource extends MemorySource { try { this.client = await mpdClient.connect({...this.clientConfig, timeout: 1000}); + this.client.on('system-player', () => { + if(this.getIsSleeping()) { + // wake up now! + this.logger.debug(`Waking up from sleeping ${Math.abs(this.getWakeAt().diff(dayjs(), 'ms'))}ms early due to player state change`) + this.setWakeAt(dayjs()); + } + }); return true; } catch (e) { let friendlyError: string | undefined; @@ -195,8 +204,16 @@ export class MPDSource extends MemorySource { getRecentlyPlayed = async (options: RecentlyPlayedOptions = {}) => { - const state = await this.client.api.status.get(); - const currentSong = await this.client.api.status.currentsong(); + let state: StatusResponse; + let currentSong: CurrentSongResponse; + try { + state = await this.client.api.status.get(); + currentSong = await this.client.api.status.currentsong(); + } catch (e) { + this.connectionOK = false; + this.authed = false; + throw e; + } let play: PlayObject | undefined; if(currentSong !== undefined) { From 706483ed061e090dd5cfdffca852d2e016d44818 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 19 Jul 2024 14:36:50 -0400 Subject: [PATCH 03/23] feat: On polling error retry check source is initialized and reinitialize if not --- src/backend/common/AbstractComponent.ts | 5 +++- src/backend/sources/AbstractSource.ts | 34 +++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 2ab477f2..af88d978 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -111,10 +111,13 @@ export default abstract class AbstractComponent { protected doAuthentication = async (): Promise => this.authed // default init function, should be overridden if auth stage is required - testAuth = async () => { + testAuth = async (force: boolean = false) => { if(!this.requiresAuth) { return; } + if(this.authed && !force) { + return; + } try { this.authed = await this.doAuthentication(); diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index a4f80576..11bea18a 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -64,6 +64,9 @@ export default abstract class AbstractSource extends AbstractComponent implement pollRetries: number = 0; tracksDiscovered: number = 0; + protected isSleeping: boolean = false; + protected wakeAt: Dayjs = dayjs(); + supportsUpstreamRecentlyPlayed: boolean = false; supportsUpstreamNowPlaying: boolean = false; @@ -305,6 +308,13 @@ export default abstract class AbstractSource extends AbstractComponent implement let pollRes: boolean | undefined = undefined; while (pollRes === undefined && this.pollRetries <= maxRetries) { try { + if(!this.isReady() && this.buildOK) { + this.logger.verbose(`Source is no longer ready! Will attempt to reinitialize => Connection OK: ${this.connectionOK} | Auth OK: ${this.authed}`); + const init = await this.initialize(); + if(init === false) { + throw new Error('Source failed reinitialization'); + } + } pollRes = await this.doPolling(); if(pollRes === true) { break; @@ -420,11 +430,13 @@ export default abstract class AbstractSource extends AbstractComponent implement } else { this.logger.debug(`Last activity was at ${this.lastActivityAt.format()} | Next check interval: ${formatNumber(sleepTime)}s`); } - const wakeUpAt = pollFrom.add(sleepTime, 'seconds'); - while(!this.shouldStopPolling() && dayjs().isBefore(wakeUpAt)) { + this.setWakeAt(pollFrom.add(sleepTime, 'seconds')); + this.setIsSleeping(true); + while(!this.shouldStopPolling() && dayjs().isBefore(this.getWakeAt())) { // check for polling status every half second and wait till wake up time await sleep(500); } + this.setIsSleeping(false); } if(this.shouldStopPolling()) { @@ -441,9 +453,27 @@ export default abstract class AbstractSource extends AbstractComponent implement this.emitEvent('statusChange', {status: 'Idle'}); this.polling = false; throw e; + } finally { + this.setIsSleeping(false); } } + protected setIsSleeping(sleeping: boolean) { + this.isSleeping = sleeping; + } + + protected getIsSleeping() { + return this.isSleeping; + } + + protected setWakeAt(dt: Dayjs) { + this.wakeAt = dt; + } + + protected getWakeAt() { + return this.wakeAt; + } + protected getInterval() { const {interval = DEFAULT_POLLING_INTERVAL} = this.config.data; return interval; From 9735b980d7c0f291d372c6594c9cdbd02b180cf6 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 19 Jul 2024 14:44:39 -0400 Subject: [PATCH 04/23] fix(tests): Do not re-use scrobbler between tests --- src/backend/tests/scrobbler/scrobblers.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index 04a3ccb1..712a4992 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -36,8 +36,6 @@ testScrobbler.verboseOptions = { }; testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); -const authScrobbler = new TestAuthScrobbler(); - describe('Networking', function () { describe('Authentication', function () { @@ -52,6 +50,7 @@ describe('Networking', function () { ) ], async function() { + const authScrobbler = new TestAuthScrobbler(); await authScrobbler.testAuth(); assert.isFalse(authScrobbler.authGated()); } @@ -66,6 +65,7 @@ describe('Networking', function () { ) ], async function() { + const authScrobbler = new TestAuthScrobbler(); await authScrobbler.testAuth(); assert.isTrue(authScrobbler.authGated()); assert.isFalse(authScrobbler.authFailure); @@ -81,6 +81,7 @@ describe('Networking', function () { ) ], async function() { + const authScrobbler = new TestAuthScrobbler(); await authScrobbler.testAuth(); assert.isTrue(authScrobbler.authGated()); assert.isTrue(authScrobbler.authFailure); From b41d7f539ffcbdaf6fe09e90ba57e4cc1c44dfd0 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 23 Jul 2024 10:48:35 -0400 Subject: [PATCH 05/23] feat(scrobble): Enable forcing existing scrobbles refresh on every scrobble #173 --- .../infrastructure/config/client/index.ts | 9 +++++++++ src/backend/common/schema/aio-client.json | 9 +++++++++ src/backend/common/schema/aio.json | 18 ++++++++++++++++++ src/backend/common/schema/client.json | 9 +++++++++ .../scrobblers/AbstractScrobbleClient.ts | 6 +++++- 5 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/backend/common/infrastructure/config/client/index.ts b/src/backend/common/infrastructure/config/client/index.ts index 58823685..667639a3 100644 --- a/src/backend/common/infrastructure/config/client/index.ts +++ b/src/backend/common/infrastructure/config/client/index.ts @@ -37,6 +37,15 @@ export interface CommonClientOptions extends RequestRetryOptions { * @examples [true] * */ refreshEnabled?: boolean + /** + * Force client to always refresh scrobbled plays from service before scrobbling new play + * + * WARNING: This will cause increased load on the scrobble service and potentially slow down scrobble speed as well. This should be used as a debugging tool and not be always-on. + * + * @default false + * @examples [false] + * */ + refreshForce?: boolean /** * The number of tracks to retrieve on initial refresh (related to scrobbleBacklogCount). If not specified this is the maximum supported for the client. diff --git a/src/backend/common/schema/aio-client.json b/src/backend/common/schema/aio-client.json index 23583d6b..0c8570d6 100644 --- a/src/backend/common/schema/aio-client.json +++ b/src/backend/common/schema/aio-client.json @@ -57,6 +57,15 @@ "title": "refreshEnabled", "type": "boolean" }, + "refreshForce": { + "default": false, + "description": "Force client to always refresh scrobbled plays from service before scrobbling new play\n\nWARNING: This will cause increased load on the scrobble service and potentially slow down scrobble speed as well. This should be used as a debugging tool and not be always-on.", + "examples": [ + false + ], + "title": "refreshForce", + "type": "boolean" + }, "refreshInitialCount": { "description": "The number of tracks to retrieve on initial refresh (related to scrobbleBacklogCount). If not specified this is the maximum supported for the client.", "title": "refreshInitialCount", diff --git a/src/backend/common/schema/aio.json b/src/backend/common/schema/aio.json index 6e62b9b8..a7be3b79 100644 --- a/src/backend/common/schema/aio.json +++ b/src/backend/common/schema/aio.json @@ -346,6 +346,15 @@ "title": "refreshEnabled", "type": "boolean" }, + "refreshForce": { + "default": false, + "description": "Force client to always refresh scrobbled plays from service before scrobbling new play\n\nWARNING: This will cause increased load on the scrobble service and potentially slow down scrobble speed as well. This should be used as a debugging tool and not be always-on.", + "examples": [ + false + ], + "title": "refreshForce", + "type": "boolean" + }, "refreshInitialCount": { "description": "The number of tracks to retrieve on initial refresh (related to scrobbleBacklogCount). If not specified this is the maximum supported for the client.", "title": "refreshInitialCount", @@ -417,6 +426,15 @@ "title": "refreshEnabled", "type": "boolean" }, + "refreshForce": { + "default": false, + "description": "Force client to always refresh scrobbled plays from service before scrobbling new play\n\nWARNING: This will cause increased load on the scrobble service and potentially slow down scrobble speed as well. This should be used as a debugging tool and not be always-on.", + "examples": [ + false + ], + "title": "refreshForce", + "type": "boolean" + }, "refreshInitialCount": { "description": "The number of tracks to retrieve on initial refresh (related to scrobbleBacklogCount). If not specified this is the maximum supported for the client.", "title": "refreshInitialCount", diff --git a/src/backend/common/schema/client.json b/src/backend/common/schema/client.json index b33f9d7a..32f402bc 100644 --- a/src/backend/common/schema/client.json +++ b/src/backend/common/schema/client.json @@ -54,6 +54,15 @@ "title": "refreshEnabled", "type": "boolean" }, + "refreshForce": { + "default": false, + "description": "Force client to always refresh scrobbled plays from service before scrobbling new play\n\nWARNING: This will cause increased load on the scrobble service and potentially slow down scrobble speed as well. This should be used as a debugging tool and not be always-on.", + "examples": [ + false + ], + "title": "refreshForce", + "type": "boolean" + }, "refreshInitialCount": { "description": "The number of tracks to retrieve on initial refresh (related to scrobbleBacklogCount). If not specified this is the maximum supported for the client.", "title": "refreshInitialCount", diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 1a673bd0..d0f7cea4 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -521,11 +521,15 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); this.logger.info('Scrobble processing started'); this.emitEvent('statusChange', {status: 'Running'}); + const { + refreshForce = false + } = this.config.options || {}; + try { this.scrobbling = true; while (!this.shouldStopScrobbleProcessing()) { while (this.queuedScrobbles.length > 0) { - if (this.lastScrobbleCheck.unix() < this.getLatestQueuePlayDate().unix()) { + if (refreshForce || (this.lastScrobbleCheck.unix() < this.getLatestQueuePlayDate().unix())) { await this.refreshScrobbles(); } const currQueuedPlay = this.queuedScrobbles.shift(); From 91e52c4c5daa964c25ae65e2af8e059ec2dfb6a1 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 23 Jul 2024 12:11:19 -0400 Subject: [PATCH 06/23] feat(musikcube): Provide more error logging and example config * Update iso-websockets to fix typing exports and get node network error from WS error * Add url example to config --- config/musikcube.json.example | 1 + package-lock.json | 18 ++++++------- package.json | 2 +- src/backend/sources/MusikcubeSource.ts | 36 +++++--------------------- 4 files changed, 18 insertions(+), 39 deletions(-) diff --git a/config/musikcube.json.example b/config/musikcube.json.example index 59435e31..1d6e6f6e 100644 --- a/config/musikcube.json.example +++ b/config/musikcube.json.example @@ -4,6 +4,7 @@ "enable": true, "name": "musikcube", "data": { + "url": "ws://localhost:7905", "password": "MY_PASSWORD" } } diff --git a/package-lock.json b/package-lock.json index 3dc6364d..ca42909e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -45,7 +45,7 @@ "fixed-size-list": "^0.3.0", "formidable": "^2.1", "gotify": "^1.1.0", - "iso-websocket": "^0.2.0", + "iso-websocket": "^0.3.0", "iti": "^0.6.0", "json5": "^2.2.3", "kodi-api": "^0.2.1", @@ -6085,15 +6085,15 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "node_modules/iso-websocket": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/iso-websocket/-/iso-websocket-0.2.0.tgz", - "integrity": "sha512-imBalzmPSq0C9CfMouimB2kZ5X1qS4Yai8kGTQdluGRb0T0iu+BkPcakFelh4FIlTM8y6+BNuCEGog3lf8HC4A==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/iso-websocket/-/iso-websocket-0.3.0.tgz", + "integrity": "sha512-RCzPkKMtX36F1FnoII4TO42aQF+ypGgtNuIamR2TwM+9a8JGBbxkxOcFf1WJKsSAh8sv1HXSjR+Lu+jGz/6oVA==", "dependencies": { - "debug": "^4.3.4", + "debug": "^4.3.5", "retry": "^0.13.1", "typescript-event-target": "^1.1.0", "unws": "^0.2.4", - "ws": "^8.16.0" + "ws": "^8.18.0" } }, "node_modules/isomorphic-ws": { @@ -10539,9 +10539,9 @@ "peer": true }, "node_modules/ws": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", - "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "engines": { "node": ">=10.0.0" }, diff --git a/package.json b/package.json index 1178db61..18524cbd 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "fixed-size-list": "^0.3.0", "formidable": "^2.1", "gotify": "^1.1.0", - "iso-websocket": "^0.2.0", + "iso-websocket": "^0.3.0", "iti": "^0.6.0", "json5": "^2.2.3", "kodi-api": "^0.2.1", diff --git a/src/backend/sources/MusikcubeSource.ts b/src/backend/sources/MusikcubeSource.ts index f92a4472..bc8094a5 100644 --- a/src/backend/sources/MusikcubeSource.ts +++ b/src/backend/sources/MusikcubeSource.ts @@ -1,9 +1,6 @@ import { childLogger } from "@foxxmd/logging"; import { EventEmitter } from "events"; -import { WS } from 'iso-websocket' -// TODO remove when/if iso-websocket exports these -// @ ts-expect-error not exported properly by package -//import { CloseEvent, ErrorEvent, RetryEvent } from "iso-websocket/dist/src/events.js"; +import { WS, CloseEvent, ErrorEvent, RetryEvent } from 'iso-websocket' import { randomUUID } from "node:crypto"; import normalizeUrl from 'normalize-url'; import pEvent from 'p-event'; @@ -102,8 +99,7 @@ export class MusikcubeSource extends MemorySource { automaticOpen: false, retry: { retries: 0 - }, - //errorInfo: true + } }); const wsLogger = childLogger(this.logger, 'WS'); this.client.addEventListener('retry', (e) => { @@ -128,12 +124,8 @@ export class MusikcubeSource extends MemorySource { this.connectionOK = false; this.authed = false; } - if(e.error.message === ('Websocket error')) { - wsLogger.error('Communication with server failed => Websocket error'); - - } else { - wsLogger.error(new Error('Communication with server failed', {cause: e.error})); - } + const hint = e.error?.cause?.message ?? undefined; + wsLogger.error(new Error(`Communication with server failed${hint !== undefined ? ` (${hint})` : ''}`, {cause: e.error})); }); this.client.addEventListener('message', (e) => { @@ -148,11 +140,12 @@ export class MusikcubeSource extends MemorySource { protected async doCheckConnection(): Promise { try { this.client.open(); - const e = await pEvent(this.client, 'open'); + const opened = await pEvent(this.client, 'open'); return true; } catch (e) { this.client.close(); - throw new Error(`Could not connect to Musikcube metadata server`); + const hint = e.error?.cause?.message ?? undefined; + throw new Error(`Could not connect to Musikcube metadata server${hint !== undefined ? ` (${hint})` : ''}`, {cause: e.error ?? e}); } } @@ -284,21 +277,6 @@ const isRetryEvent = (e: Event): e is RetryEvent => { return e.type === 'retry'; } -// TODO remove when/if iso-websockets exports these -interface ErrorEvent extends Event { - type: 'error' - error: Error - message: string -} -interface CloseEvent extends Event { - type: 'close' - reason: string - code: number -} -interface RetryEvent extends Event { - type: 'retry' -} - const isAuthenticateResponse = (data: any): data is MCAuthenticateResponse => { return 'name' in data && data.name === 'authenticate'; } From 9c2bf05a243bb480fbbf991f0c80b33a3ccbe6b8 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 24 Jul 2024 10:31:37 -0400 Subject: [PATCH 07/23] refactor(listenbrainz): Simplify scrobble api calls and logging --- .../common/vendor/ListenbrainzApiClient.ts | 35 ++++++++++++++++--- .../scrobblers/ListenbrainzScrobbler.ts | 16 ++------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/backend/common/vendor/ListenbrainzApiClient.ts b/src/backend/common/vendor/ListenbrainzApiClient.ts index 1e85eb2f..94f4851b 100644 --- a/src/backend/common/vendor/ListenbrainzApiClient.ts +++ b/src/backend/common/vendor/ListenbrainzApiClient.ts @@ -1,6 +1,6 @@ import { stringSameness } from '@foxxmd/string-sameness'; import dayjs from "dayjs"; -import request, { Request } from 'superagent'; +import request, { Request, Response } from 'superagent'; import { PlayObject } from "../../../core/Atomic.js"; import { slice } from "../../../core/StringUtils.js"; import { combinePartsToString } from "../../utils.js"; @@ -127,7 +127,7 @@ export class ListenbrainzApiClient extends AbstractApiClient { } - callApi = async (req: Request, retries = 0): Promise => { + callApi = async (req: Request, retries = 0): Promise => { const { maxRequestRetries = 2, retryMultiplier = DEFAULT_RETRY_MULTIPLIER @@ -248,17 +248,27 @@ export class ListenbrainzApiClient extends AbstractApiClient { } } - submitListen = async (play: PlayObject) => { + submitListen = async (play: PlayObject, log: boolean = false) => { try { const listenPayload: SubmitPayload = {listen_type: 'single', payload: [ListenbrainzApiClient.playToListenPayload(play)]}; - await this.callApi(request.post(`${this.url}1/submit-listens`).type('json').send(listenPayload)); + if(log) { + this.logger.debug(`Submit Payload: ${JSON.stringify(listenPayload)}`); + } + // response consists of {"status": "ok"} + // so no useful information + // https://listenbrainz.readthedocs.io/en/latest/users/api-usage.html#submitting-listens + // TODO may we should make a call to recent-listens to get the parsed scrobble? + const resp = await this.callApi(request.post(`${this.url}1/submit-listens`).type('json').send(listenPayload)); + if(log) { + this.logger.debug(`Submit Response: ${resp.text}`) + } return listenPayload; } catch (e) { throw e; } } - static playToListenPayload = (play: PlayObject): ListenPayload => { + static playToListenPayload(play: PlayObject): ListenPayload { const { data: { playDate, @@ -581,6 +591,21 @@ export class ListenbrainzApiClient extends AbstractApiClient { } } + static submitToPlayObj(submitObj: SubmitPayload, playObj: PlayObject): PlayObject { + if (submitObj.payload.length > 0) { + const respPlay = { + ...playObj, + }; + respPlay.data = { + ...playObj.data, + album: submitObj.payload[0].track_metadata?.release_name ?? playObj.data.album, + track: submitObj.payload[0].track_metadata?.track_name ?? playObj.data.album, + }; + return respPlay; + } + return playObj; + } + static formatPlayObj(obj: any, options: FormatPlayObjectOptions): PlayObject { return ListenbrainzApiClient.listenResponseToPlay(obj); } diff --git a/src/backend/scrobblers/ListenbrainzScrobbler.ts b/src/backend/scrobblers/ListenbrainzScrobbler.ts index 15ddd6b1..24cd9f6e 100644 --- a/src/backend/scrobblers/ListenbrainzScrobbler.ts +++ b/src/backend/scrobblers/ListenbrainzScrobbler.ts @@ -91,30 +91,18 @@ export default class ListenbrainzScrobbler extends AbstractScrobbleClient { } = {} } = playObj; - let rawPayload = {listen_type: 'single', payload: [this.playToClientPayload(playObj)]}; - try { - const resp = await this.api.submitListen(playObj); - rawPayload = resp; + await this.api.submitListen(playObj, true); if (newFromSource) { this.logger.info(`Scrobbled (New) => (${source}) ${buildTrackString(playObj)}`); } else { this.logger.info(`Scrobbled (Backlog) => (${source}) ${buildTrackString(playObj)}`); } - // last fm has rate limits but i can't find a specific example of what that limit is. going to default to 1 scrobble/sec to be safe - //await sleep(1000); return playObj; } catch (e) { await this.notifier.notify({title: `Client - ${capitalize(this.type)} - ${this.name} - Scrobble Error`, message: `Failed to scrobble => ${buildTrackString(playObj)} | Error: ${e.message}`, priority: 'error'}); - this.logger.error(`Failed to scrobble => ${e.message}`, {payload: rawPayload}); - if(e instanceof UpstreamError) { - throw e; - } else { - throw new UpstreamError(`Error occurred while making Listenbrainz API request: ${e.message}`, {cause: e, showStopper: true}); - } - } finally { - this.logger.debug(`Raw Payload:`, {rawPayload}); + throw new UpstreamError(`Error occurred while making Listenbrainz API scrobble request: ${e.message}`, {cause: e, showStopper: !(e instanceof UpstreamError)}); } } } From d9cd3764b95e8d6e52829712fab6af7705471c9d Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 24 Jul 2024 11:20:09 -0400 Subject: [PATCH 08/23] refactor(scrobbler): Consolidate refresh logic and add more logging * Move recentScrobbles and generic refresh logic into abstract class * New abstract function for scrobbler children to implement to get scrobbler-specific play lists on refresh * Implement shouldRefereshScrobble with logging for better insight into why or why not a scrobbler refreshed upstream scrobbles --- .../scrobblers/AbstractScrobbleClient.ts | 59 ++++++++++++++++--- src/backend/scrobblers/LastfmScrobbler.ts | 17 +----- .../scrobblers/ListenbrainzScrobbler.ts | 18 +----- src/backend/scrobblers/MalojaScrobbler.ts | 31 +++------- src/backend/tests/scrobbler/TestScrobbler.ts | 3 + .../tests/scrobbler/scrobblers.test.ts | 5 +- 6 files changed, 71 insertions(+), 62 deletions(-) diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index d0f7cea4..e811d318 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -148,10 +148,25 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i await this.refreshScrobbles(initialLimit); } - refreshScrobbles = async (limit?: number) => { - this.logger.debug('Scrobbler does not have refresh function implemented!'); + refreshScrobbles = async (limit: number = this.MAX_STORED_SCROBBLES) => { + if (this.refreshEnabled) { + this.logger.debug('Refreshing recent scrobbles'); + const recent = await this.getScrobblesForRefresh(limit); + this.logger.debug(`Found ${recent.length} recent scrobbles`); + if (this.recentScrobbles.length > 0) { + const [{data: {playDate: newestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(-1); + const [{data: {playDate: oldestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(0, 1); + this.newestScrobbleTime = newestScrobbleTime; + this.oldestScrobbleTime = oldestScrobbleTime; + + this.filterScrobbledTracks(); + } + } + this.lastScrobbleCheck = dayjs(); } + protected abstract getScrobblesForRefresh(limit: number): Promise; + public abstract alreadyScrobbled(playObj: PlayObject, log?: boolean): Promise; scrobblesLastCheckedAt = () => this.lastScrobbleCheck @@ -514,6 +529,40 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); protected shouldStopScrobbleProcessing = () => this.scrobbling === false || this.userScrobblingStopSignal !== undefined; + shouldRefreshScrobble = (ignoreForce: boolean = false) => { + const { + refreshForce = false + } = this.config.options || {}; + + if (!this.refreshEnabled) { + this.logger.debug(`Should NOT refresh scrobbles => refreshEnabled is false`); + return false; + } + + if(refreshForce && !ignoreForce) { + this.logger.debug(`Should refresh scrobbles => refreshForce is true`); + return true; + } + + const queuedPlayedDate = this.getLatestQueuePlayDate(); + + // if next queued play was played more recently than the last time we refreshed upstream scrobbles + if (this.lastScrobbleCheck.unix() < queuedPlayedDate.unix()) { + this.logger.debug('Should refresh scrobbles => queued scrobble playDate is newer than last upstream scrobble refresh'); + return true; + } + + // if the last scrobbled play is at or is newer than the next scrobble then we are inserting (or potentially duping) + // in which case our data is probably stale + if(this.newestScrobbleTime !== undefined && this.newestScrobbleTime.unix() >= queuedPlayedDate.unix()) { + this.logger.debug('Should refresh scrobbles => queued scrobble playDate is equal to or older than the newest upstream scrobble'); + return true; + } + + this.logger.debug('Scrobble refresh not needed'); + return false; + } + protected doProcessing = async (): Promise => { if (this.scrobbling === true) { return true; @@ -521,15 +570,11 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); this.logger.info('Scrobble processing started'); this.emitEvent('statusChange', {status: 'Running'}); - const { - refreshForce = false - } = this.config.options || {}; - try { this.scrobbling = true; while (!this.shouldStopScrobbleProcessing()) { while (this.queuedScrobbles.length > 0) { - if (refreshForce || (this.lastScrobbleCheck.unix() < this.getLatestQueuePlayDate().unix())) { + if (this.shouldRefreshScrobble()) { await this.refreshScrobbles(); } const currQueuedPlay = this.queuedScrobbles.shift(); diff --git a/src/backend/scrobblers/LastfmScrobbler.ts b/src/backend/scrobblers/LastfmScrobbler.ts index 0ccf21fe..f7f0f5a3 100644 --- a/src/backend/scrobblers/LastfmScrobbler.ts +++ b/src/backend/scrobblers/LastfmScrobbler.ts @@ -46,9 +46,7 @@ export default class LastfmScrobbler extends AbstractScrobbleClient { } } - refreshScrobbles = async (limit = this.MAX_STORED_SCROBBLES) => { - if (this.refreshEnabled) { - this.logger.debug('Refreshing recent scrobbles'); + getScrobblesForRefresh = async (limit: number) => { const resp = await this.api.callApi((client: any) => client.userGetRecentTracks({ user: this.api.user, sk: this.api.client.sessionKey, @@ -60,7 +58,7 @@ export default class LastfmScrobbler extends AbstractScrobbleClient { track: list = [], } } = resp; - this.recentScrobbles = list.reduce((acc: any, x: any) => { + return list.reduce((acc: any, x: any) => { try { const formatted = LastfmApiClient.formatPlayObj(x); const { @@ -91,17 +89,6 @@ export default class LastfmScrobbler extends AbstractScrobbleClient { return acc; } }, []); - this.logger.debug(`Found ${this.recentScrobbles.length} recent scrobbles`); - if (this.recentScrobbles.length > 0) { - const [{data: {playDate: newestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(-1); - const [{data: {playDate: oldestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(0, 1); - this.newestScrobbleTime = newestScrobbleTime; - this.oldestScrobbleTime = oldestScrobbleTime; - - this.filterScrobbledTracks(); - } - } - this.lastScrobbleCheck = dayjs(); } cleanSourceSearchTitle = (playObj: PlayObject) => { diff --git a/src/backend/scrobblers/ListenbrainzScrobbler.ts b/src/backend/scrobblers/ListenbrainzScrobbler.ts index 24cd9f6e..e63117e5 100644 --- a/src/backend/scrobblers/ListenbrainzScrobbler.ts +++ b/src/backend/scrobblers/ListenbrainzScrobbler.ts @@ -59,22 +59,8 @@ export default class ListenbrainzScrobbler extends AbstractScrobbleClient { } } - refreshScrobbles = async (limit = this.MAX_STORED_SCROBBLES) => { - if (this.refreshEnabled) { - this.logger.debug('Refreshing recent scrobbles'); - const resp = await this.api.getRecentlyPlayed(limit); - this.logger.debug(`Found ${resp.length} recent scrobbles`); - this.recentScrobbles = resp; - if (this.recentScrobbles.length > 0) { - const [{data: {playDate: newestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(-1); - const [{data: {playDate: oldestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(0, 1); - this.newestScrobbleTime = newestScrobbleTime; - this.oldestScrobbleTime = oldestScrobbleTime; - - this.filterScrobbledTracks(); - } - } - this.lastScrobbleCheck = dayjs(); + getScrobblesForRefresh = async (limit: number) => { + return await this.api.getRecentlyPlayed(limit); } alreadyScrobbled = async (playObj: PlayObject, log = false) => (await this.existingScrobble(playObj)) !== undefined diff --git a/src/backend/scrobblers/MalojaScrobbler.ts b/src/backend/scrobblers/MalojaScrobbler.ts index bf7398e6..c66b4c95 100644 --- a/src/backend/scrobblers/MalojaScrobbler.ts +++ b/src/backend/scrobblers/MalojaScrobbler.ts @@ -299,28 +299,15 @@ export default class MalojaScrobbler extends AbstractScrobbleClient { } } - refreshScrobbles = async (limit = this.MAX_STORED_SCROBBLES) => { - if (this.refreshEnabled) { - this.logger.debug('Refreshing recent scrobbles'); - const {url} = this.config.data; - const resp = await this.callApi(request.get(`${url}/apis/mlj_1/scrobbles?perpage=${limit}`)); - const { - body: { - list = [], - } = {}, - } = resp; - this.logger.debug(`Found ${list.length} recent scrobbles`); - this.recentScrobbles = list.map((x: any) => this.formatPlayObj(x)); - if (this.recentScrobbles.length > 0) { - const [{data: {playDate: newestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(-1); - const [{data: {playDate: oldestScrobbleTime = dayjs()} = {}} = {}] = this.recentScrobbles.slice(0, 1); - this.newestScrobbleTime = newestScrobbleTime; - this.oldestScrobbleTime = oldestScrobbleTime; - - this.filterScrobbledTracks(); - } - } - this.lastScrobbleCheck = dayjs(); + getScrobblesForRefresh = async (limit: number) => { + const {url} = this.config.data; + const resp = await this.callApi(request.get(`${url}/apis/mlj_1/scrobbles?perpage=${limit}`)); + const { + body: { + list = [], + } = {}, + } = resp; + return list.map((x: any) => this.formatPlayObj(x)); } cleanSourceSearchTitle = (playObj: PlayObject) => { diff --git a/src/backend/tests/scrobbler/TestScrobbler.ts b/src/backend/tests/scrobbler/TestScrobbler.ts index 297edb4b..7006063e 100644 --- a/src/backend/tests/scrobbler/TestScrobbler.ts +++ b/src/backend/tests/scrobbler/TestScrobbler.ts @@ -6,6 +6,9 @@ import { Notifiers } from "../../notifier/Notifiers.js"; import AbstractScrobbleClient from "../../scrobblers/AbstractScrobbleClient.js"; export class TestScrobbler extends AbstractScrobbleClient { + protected async getScrobblesForRefresh(limit: number): Promise { + return []; + } constructor() { const logger = loggerTest; diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index 712a4992..22503d99 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -427,12 +427,13 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble describe('Manages scrobble queue', function() { - before(function() { + before(async function() { + await testScrobbler.initialize(); testScrobbler.recentScrobbles = normalizedWithMixedDur; testScrobbler.scrobbleSleep = 500; testScrobbler.scrobbleDelay = 0; testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); - testScrobbler.initScrobbleMonitoring(); + testScrobbler.initScrobbleMonitoring().catch(console.error); }); it('Scrobbles a uniquely queued play', async function() { From e907892c4af78b5a8b5e0982bb88c1b6f17e103f Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Wed, 24 Jul 2024 12:06:20 -0400 Subject: [PATCH 09/23] refactor(scrobbler): Change refresh force to user-configurable staleness time Use refreshStaleAfter to force refresh to user can configure the time. Allows more nuanced refreshing behavior as well as always refresh (0 seconds) --- .../infrastructure/config/client/index.ts | 11 +-- .../scrobblers/AbstractScrobbleClient.ts | 71 ++++++++++--------- .../tests/scrobbler/scrobblers.test.ts | 59 +++++++++++++++ 3 files changed, 102 insertions(+), 39 deletions(-) diff --git a/src/backend/common/infrastructure/config/client/index.ts b/src/backend/common/infrastructure/config/client/index.ts index 667639a3..3699467c 100644 --- a/src/backend/common/infrastructure/config/client/index.ts +++ b/src/backend/common/infrastructure/config/client/index.ts @@ -38,14 +38,15 @@ export interface CommonClientOptions extends RequestRetryOptions { * */ refreshEnabled?: boolean /** - * Force client to always refresh scrobbled plays from service before scrobbling new play + * Force client to refresh scrobbled plays from upstream service if last refresh was at least X seconds ago * - * WARNING: This will cause increased load on the scrobble service and potentially slow down scrobble speed as well. This should be used as a debugging tool and not be always-on. + * **In most case this setting should NOT be used.** MS intelligently refreshes based on activity so using this setting may increase upstream service load and slow down scrobbles. * - * @default false - * @examples [false] + * This setting should only be used in specific scenarios where MS is handling multiple "relaying" client-services (IE lfm -> lz -> lfm) and there is the potential for a client to be out of sync after more than a few seconds. + * + * @examples [3] * */ - refreshForce?: boolean + refreshStaleAfter?: number /** * The number of tracks to retrieve on initial refresh (related to scrobbleBacklogCount). If not specified this is the maximum supported for the client. diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index e811d318..627ea495 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -167,6 +167,43 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i protected abstract getScrobblesForRefresh(limit: number): Promise; + shouldRefreshScrobble = () => { + const { + refreshStaleAfter + } = this.config.options || {}; + + if (!this.refreshEnabled) { + this.logger.debug(`Should NOT refresh scrobbles => refreshEnabled is false`); + return false; + } + + const queuedPlayedDate = this.getLatestQueuePlayDate(); + + // if next queued play was played more recently than the last time we refreshed upstream scrobbles + if (this.lastScrobbleCheck.unix() < queuedPlayedDate.unix()) { + this.logger.debug('Should refresh scrobbles => queued scrobble playDate is newer than last upstream scrobble refresh'); + return true; + } + + // if the last scrobbled play is at or is newer than the next scrobble then we are inserting (or potentially duping) + // in which case our data is probably stale + if(this.newestScrobbleTime !== undefined && this.newestScrobbleTime.unix() >= queuedPlayedDate.unix()) { + this.logger.debug('Should refresh scrobbles => queued scrobble playDate is equal to or older than the newest upstream scrobble'); + return true; + } + + if(refreshStaleAfter !== undefined) { + const diff = dayjs().diff(this.lastScrobbleCheck, 's'); + if(diff > refreshStaleAfter) { + this.logger.debug(`Should refresh scrobbles => last refresh (${diff}s ago) was longer than refreshStaleAfter (${refreshStaleAfter}s)`); + return true; + } + } + + this.logger.debug('Scrobble refresh not needed'); + return false; + } + public abstract alreadyScrobbled(playObj: PlayObject, log?: boolean): Promise; scrobblesLastCheckedAt = () => this.lastScrobbleCheck @@ -529,40 +566,6 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); protected shouldStopScrobbleProcessing = () => this.scrobbling === false || this.userScrobblingStopSignal !== undefined; - shouldRefreshScrobble = (ignoreForce: boolean = false) => { - const { - refreshForce = false - } = this.config.options || {}; - - if (!this.refreshEnabled) { - this.logger.debug(`Should NOT refresh scrobbles => refreshEnabled is false`); - return false; - } - - if(refreshForce && !ignoreForce) { - this.logger.debug(`Should refresh scrobbles => refreshForce is true`); - return true; - } - - const queuedPlayedDate = this.getLatestQueuePlayDate(); - - // if next queued play was played more recently than the last time we refreshed upstream scrobbles - if (this.lastScrobbleCheck.unix() < queuedPlayedDate.unix()) { - this.logger.debug('Should refresh scrobbles => queued scrobble playDate is newer than last upstream scrobble refresh'); - return true; - } - - // if the last scrobbled play is at or is newer than the next scrobble then we are inserting (or potentially duping) - // in which case our data is probably stale - if(this.newestScrobbleTime !== undefined && this.newestScrobbleTime.unix() >= queuedPlayedDate.unix()) { - this.logger.debug('Should refresh scrobbles => queued scrobble playDate is equal to or older than the newest upstream scrobble'); - return true; - } - - this.logger.debug('Scrobble refresh not needed'); - return false; - } - protected doProcessing = async (): Promise => { if (this.scrobbling === true) { return true; diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index 22503d99..6749e080 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -425,6 +425,65 @@ describe('Detects duplicate and unique scrobbles using actively tracked scrobble }); }); +describe('Detects when upstream scrobbles should be refreshed', function() { + + const normalizedClose = normalizePlays(withDurPlays, {initialDate: dayjs().subtract(100, 'seconds')}); + + beforeEach(function () { + testScrobbler.recentScrobbles = normalizedWithMixedDur; + testScrobbler.newestScrobbleTime = normalizedWithMixedDur[0].data.playDate; + testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); + testScrobbler.queuedScrobbles = []; + testScrobbler.config.options = {}; + }); + + it('Detects queued scrobble date is newer than last scrobble refresh', async function() { + const newScrobble = generatePlay({ + playDate: dayjs() + }); + + testScrobbler.queueScrobble(newScrobble, 'test'); + assert.isTrue(testScrobbler.shouldRefreshScrobble()); + }); + + it('Detects queued scrobble date is older than newest scrobble', async function() { + testScrobbler.recentScrobbles = normalizedClose; + testScrobbler.newestScrobbleTime = normalizedClose[0].data.playDate; + + const newScrobble = generatePlay({ + playDate: dayjs().subtract(120, 'seconds') + }); + + testScrobbler.queueScrobble(newScrobble, 'test'); + assert.isTrue(testScrobbler.shouldRefreshScrobble()); + }); + + it('Forces refresh if refreshStaleAfter is set', async function() { + testScrobbler.recentScrobbles = normalizedClose; + testScrobbler.newestScrobbleTime = normalizedClose[0].data.playDate; + testScrobbler.config.options = { refreshStaleAfter: 10 }; + + const newScrobble = generatePlay({ + playDate: dayjs().subtract(80, 'seconds') + }); + + testScrobbler.queueScrobble(newScrobble, 'test'); + assert.isTrue(testScrobbler.shouldRefreshScrobble()); + }); + + it('Does not refresh if scrobble is older than last check but newer than newest upstream scrobble', async function() { + testScrobbler.recentScrobbles = normalizedClose; + testScrobbler.newestScrobbleTime = normalizedClose[0].data.playDate; + + const newScrobble = generatePlay({ + playDate: dayjs().subtract(80, 'seconds') + }); + + testScrobbler.queueScrobble(newScrobble, 'test'); + assert.isFalse(testScrobbler.shouldRefreshScrobble()); + }); +}); + describe('Manages scrobble queue', function() { before(async function() { From 2ae82fdd9e79991bd90307dd4b91c938aaccb4d7 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 25 Jul 2024 13:43:34 -0400 Subject: [PATCH 10/23] feat: Implement common play transform functionality --- package-lock.json | 195 ++++++++++++++++++ package.json | 1 + src/backend/common/AbstractComponent.ts | 195 ++++++++++++++++++ src/backend/common/infrastructure/Atomic.ts | 28 +++ .../infrastructure/config/client/index.ts | 4 +- .../common/infrastructure/config/common.ts | 4 +- .../infrastructure/config/source/index.ts | 4 +- .../scrobblers/AbstractScrobbleClient.ts | 5 +- src/backend/sources/AbstractSource.ts | 4 +- src/backend/tests/component/component.test.ts | 172 +++++++++++++++ src/backend/utils.ts | 66 +++++- 11 files changed, 668 insertions(+), 10 deletions(-) create mode 100644 src/backend/tests/component/component.test.ts diff --git a/package-lock.json b/package-lock.json index ca42909e..9570a48f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@foxxmd/chromecast-client": "^1.0.4", "@foxxmd/get-version": "^0.0.3", "@foxxmd/logging": "^0.1.14", + "@foxxmd/regex-buddy-core": "^0.1.0", "@foxxmd/string-sameness": "^0.4.0", "@kenyip/backoff-strategies": "^1.0.4", "@react-nano/use-event-source": "^0.13.0", @@ -1221,6 +1222,18 @@ "npm": ">=9.3.0" } }, + "node_modules/@foxxmd/regex-buddy-core": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@foxxmd/regex-buddy-core/-/regex-buddy-core-0.1.0.tgz", + "integrity": "sha512-CzmFvnbl2mWnH4jJZzBIe6HwLCKFfahHMKUvj3IB6bCnHn0yTO2BKVpm6tqTvIw8BNXqggbiONp5mMLe8DroLA==", + "dependencies": { + "@stdlib/regexp-regexp": "^0.2.1" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=9.3.0" + } + }, "node_modules/@foxxmd/string-sameness": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@foxxmd/string-sameness/-/string-sameness-0.4.0.tgz", @@ -2015,6 +2028,188 @@ "url": "https://github.com/sindresorhus/is?sponsor=1" } }, + "node_modules/@stdlib/error-tools-fmtprodmsg": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@stdlib/error-tools-fmtprodmsg/-/error-tools-fmtprodmsg-0.2.1.tgz", + "integrity": "sha512-SaxvGeGfWfda/O3rTNGRGBzAL9gsY/yd8n1hXwzOl/2aUHf8nxcf6Fz6/BQ5PguT0GiBkca19XEhHZZHxX3X/g==", + "os": [ + "aix", + "darwin", + "freebsd", + "linux", + "macos", + "openbsd", + "sunos", + "win32", + "windows" + ], + "engines": { + "node": ">=0.10.0", + "npm": ">2.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/stdlib" + } + }, + "node_modules/@stdlib/regexp-regexp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@stdlib/regexp-regexp/-/regexp-regexp-0.2.1.tgz", + "integrity": "sha512-f25kXWc73YVPxDxC8/25NM6AKHn3gH5WVcQfhjyy1FMM++XLNFy2M+1sdAyQ8PnlZ9qJwtn3/NjbACeETGfYHQ==", + "os": [ + "aix", + "darwin", + "freebsd", + "linux", + "macos", + "openbsd", + "sunos", + "win32", + "windows" + ], + "dependencies": { + "@stdlib/utils-define-nonenumerable-read-only-property": "^0.2.1" + }, + "engines": { + "node": ">=0.10.0", + "npm": ">2.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/stdlib" + } + }, + "node_modules/@stdlib/string-base-format-interpolate": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@stdlib/string-base-format-interpolate/-/string-base-format-interpolate-0.2.1.tgz", + "integrity": "sha512-Uxz89eUi4m9yao4VjsqXIxLIF7qDmqEAH0e+XBRWRGC2zx6DhmK2kLnaU0xW69+VJPn3dq4itxq0oryw2E+qIQ==", + "os": [ + "aix", + "darwin", + "freebsd", + "linux", + "macos", + "openbsd", + "sunos", + "win32", + "windows" + ], + "engines": { + "node": ">=0.10.0", + "npm": ">2.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/stdlib" + } + }, + "node_modules/@stdlib/string-base-format-tokenize": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@stdlib/string-base-format-tokenize/-/string-base-format-tokenize-0.2.1.tgz", + "integrity": "sha512-3Ut96pmCgEFArrdwXKm1q0j1FOqTnG/uOsh24uYNU/ABRsMOOajRlAjCCdQv9f8P916qPrSnF1V3Pd18LAaksg==", + "os": [ + "aix", + "darwin", + "freebsd", + "linux", + "macos", + "openbsd", + "sunos", + "win32", + "windows" + ], + "engines": { + "node": ">=0.10.0", + "npm": ">2.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/stdlib" + } + }, + "node_modules/@stdlib/string-format": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@stdlib/string-format/-/string-format-0.2.1.tgz", + "integrity": "sha512-+HpXkEJ0Z4gthH5KicXvRRJiCiCTSrKzM+mS8N6vwaAD+OG+Oq8Cn43XBD1ic/UHROI9un42MruF1ZLlkSmdOw==", + "os": [ + "aix", + "darwin", + "freebsd", + "linux", + "macos", + "openbsd", + "sunos", + "win32", + "windows" + ], + "dependencies": { + "@stdlib/string-base-format-interpolate": "^0.2.1", + "@stdlib/string-base-format-tokenize": "^0.2.1" + }, + "engines": { + "node": ">=0.10.0", + "npm": ">2.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/stdlib" + } + }, + "node_modules/@stdlib/utils-define-nonenumerable-read-only-property": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@stdlib/utils-define-nonenumerable-read-only-property/-/utils-define-nonenumerable-read-only-property-0.2.1.tgz", + "integrity": "sha512-L8fs1kI79T2RQIg8rHR9aQnnSDELqiDGWbK3jA1NP8iW+ydxlxXyO8Dw17fBCXVua3Y19a1NVyGtIN5WGe2UCw==", + "os": [ + "aix", + "darwin", + "freebsd", + "linux", + "macos", + "openbsd", + "sunos", + "win32", + "windows" + ], + "dependencies": { + "@stdlib/utils-define-property": "^0.2.1" + }, + "engines": { + "node": ">=0.10.0", + "npm": ">2.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/stdlib" + } + }, + "node_modules/@stdlib/utils-define-property": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@stdlib/utils-define-property/-/utils-define-property-0.2.3.tgz", + "integrity": "sha512-+EzWImaQR/6XNFbXIITFi3PLQGTbKVIWSYxJfHXAuTtibAMnhHOWvEzKOumVe/Q4Cdsrc3/PIkpjJzliqAX9AA==", + "os": [ + "aix", + "darwin", + "freebsd", + "linux", + "macos", + "openbsd", + "sunos", + "win32", + "windows" + ], + "dependencies": { + "@stdlib/error-tools-fmtprodmsg": "^0.2.1", + "@stdlib/string-format": "^0.2.1" + }, + "engines": { + "node": ">=0.10.0", + "npm": ">2.7.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/stdlib" + } + }, "node_modules/@supercharge/promise-pool": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@supercharge/promise-pool/-/promise-pool-3.2.0.tgz", diff --git a/package.json b/package.json index 18524cbd..d9d4aae4 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@foxxmd/chromecast-client": "^1.0.4", "@foxxmd/get-version": "^0.0.3", "@foxxmd/logging": "^0.1.14", + "@foxxmd/regex-buddy-core": "^0.1.0", "@foxxmd/string-sameness": "^0.4.0", "@kenyip/backoff-strategies": "^1.0.4", "@react-nano/use-event-source": "^0.13.0", diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index af88d978..3a45b630 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -1,6 +1,13 @@ import { Logger } from "@foxxmd/logging"; +import { searchAndReplace, SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core"; +import { compare } from "compare-versions"; +import { PlayObject } from "../../core/Atomic.js"; +import { configPartsToStrongParts, configValToSearchReplace } from "../utils.js"; import { hasNodeNetworkException } from "./errors/NodeErrors.js"; import { hasUpstreamError } from "./errors/UpstreamError.js"; +import { PlayTransformParts, PlayTransformRules, TRANSFORM_HOOK, TransformHook } from "./infrastructure/Atomic.js"; +import { CommonClientConfig } from "./infrastructure/config/client/index.js"; +import { CommonSourceConfig } from "./infrastructure/config/source/index.js"; export default abstract class AbstractComponent { requiresAuth: boolean = false; @@ -13,13 +20,22 @@ export default abstract class AbstractComponent { initializing: boolean = false; + config: CommonClientConfig | CommonSourceConfig; + + transformRules!: PlayTransformRules; + logger: Logger; + protected constructor(config: CommonClientConfig | CommonSourceConfig) { + this.config = config; + } + initialize = async () => { this.logger.debug('Attempting to initialize...'); try { this.initializing = true; await this.buildInitData(); + this.buildTransformRules(); await this.checkConnection(); await this.testAuth(); this.logger.info('Fully Initialized!'); @@ -72,6 +88,74 @@ export default abstract class AbstractComponent { return; } + public buildTransformRules() { + try { + this.doBuildTransformRules(); + } catch (e) { + this.buildOK = false; + throw new Error('Could not build playTransform rules. Check your configuration is valid.', {cause: e}); + } + } + + protected doBuildTransformRules() { + const { + options: { + playTransform + } = {} + } = this.config; + + if (playTransform === undefined) { + this.transformRules = {}; + return; + } + + const { + preCompare: preConfig, + compare: { + candidate: candidateConfig, + existing: existingConfig, + } = {}, + postCompare: postConfig + } = playTransform; + + let preCompare, + candidate, + existing, + postCompare; + + try { + preCompare = configPartsToStrongParts(preConfig) + } catch (e) { + throw new Error('preCompare was not valid', {cause: e}); + } + + try { + candidate = configPartsToStrongParts(candidateConfig) + } catch (e) { + throw new Error('candidate was not valid', {cause: e}); + } + + try { + existing = configPartsToStrongParts(existingConfig) + } catch (e) { + throw new Error('existing was not valid', {cause: e}); + } + + try { + postCompare = configPartsToStrongParts(postConfig) + } catch (e) { + throw new Error('postCompare was not valid', {cause: e}); + } + + this.transformRules = { + preCompare, + compare: { + candidate, + existing, + }, + postCompare, + } + } public async checkConnection() { try { @@ -149,4 +233,115 @@ export default abstract class AbstractComponent { protected async postInitialize(): Promise { return; } + + public transformPlay = (play: PlayObject, hookType: TransformHook, log: boolean = true) => { + let hook: PlayTransformParts | undefined; + + switch (hookType) { + case TRANSFORM_HOOK.preCompare: + hook = this.transformRules.preCompare; + break; + case TRANSFORM_HOOK.candidate: + hook = this.transformRules.compare?.candidate; + break; + case TRANSFORM_HOOK.existing: + hook = this.transformRules.compare?.candidate; + break; + case TRANSFORM_HOOK.postCompare: + hook = this.transformRules.postCompare; + break; + } + + if (hook === undefined) { + return play; + } + + const { + data: { + track, + artists, + albumArtists, + album + } = {} + } = play; + + const transformedPlay = { + ...play + } + + const results: [string, string, string][] = []; + + if (hook.title !== undefined && track !== undefined) { + try { + const t = searchAndReplace(track, hook.title); + if (t !== track) { + results.push(['title', track, t]); + transformedPlay.data.track = t; + } + } catch (e) { + this.logger.warn(`Failed to transform title during ${hookType} transform: ${track}`, {cause: e}); + } + } + + if (hook.artists !== undefined && artists !== undefined && artists.length > 0) { + const transformedArtists: string[] = []; + let anyTransformed = false; + for (const artist of artists) { + try { + const t = searchAndReplace(artist, hook.artists); + if (t !== artist) { + anyTransformed = true; + } + transformedArtists.push(t); + } catch (e) { + this.logger.warn(`Failed to transform artist during ${hookType} transform: ${artist}`, {cause: e}); + transformedArtists.push(artist); + } + } + transformedPlay.data.artists = transformedArtists; + if (anyTransformed) { + results.push(['artists', artists.join(' / '), transformedArtists.join(' / ')]); + } + } + + if (hook.artists !== undefined && albumArtists !== undefined && albumArtists.length > 0) { + const transformedArtists: string[] = []; + let anyTransformed = false; + for (const artist of albumArtists) { + try { + const t = searchAndReplace(artist, hook.artists); + if (t !== artist) { + anyTransformed = true; + } + transformedArtists.push(t); + } catch (e) { + this.logger.warn(`Failed to transform albumArtist during ${hookType} transform: ${artist}`, {cause: e}); + transformedArtists.push(artist); + } + } + transformedPlay.data.albumArtists = transformedArtists; + if (anyTransformed) { + results.push(['albumArtists', artists.join(' / '), transformedArtists.join(' / ')]); + } + } + + if (hook.album !== undefined && album !== undefined) { + try { + const t = searchAndReplace(album, hook.album); + if (t !== album) { + results.push(['album', album, t]); + transformedPlay.data.album = t; + } + } catch (e) { + this.logger.warn(`Failed to transform album during ${hookType} transform: ${album}`, {cause: e}); + } + } + + if(results.length > 0) { + this.logger.debug(`Play transformed by ${hookType}: + ${results.map(x => `${x[0]}: ${x[1]} => ${x[2]}`).join('\n')}`); + } + + return transformedPlay; + } } diff --git a/src/backend/common/infrastructure/Atomic.ts b/src/backend/common/infrastructure/Atomic.ts index 897aa747..1ae13a7c 100644 --- a/src/backend/common/infrastructure/Atomic.ts +++ b/src/backend/common/infrastructure/Atomic.ts @@ -1,4 +1,5 @@ import { Logger } from '@foxxmd/logging'; +import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core"; import { Dayjs } from "dayjs"; import { Request, Response } from "express"; import { NextFunction, ParamsDictionary, Query } from "express-serve-static-core"; @@ -240,3 +241,30 @@ export interface MdnsDeviceInfo { export type AbstractApiOptions = Record & { logger: Logger } export type keyOmit = T & { [P in U]?: never } + +export type SearchAndReplaceTerm = string | SearchAndReplaceRegExp; + +export interface PlayTransformParts { + title?: T[] + artists?: T[] + album?: T[] +} + +export interface PlayTransformHooks { + preCompare?: PlayTransformParts + compare?: { + candidate?: PlayTransformParts + existing?: PlayTransformParts + } + postCompare?: PlayTransformParts +} + +export type PlayTransformRules = PlayTransformHooks + +export type TransformHook = 'preCompare' | 'compare' | 'candidate' | 'existing' | 'postCompare'; +export const TRANSFORM_HOOK = { + preCompare: 'preCompare' as TransformHook, + candidate: 'candidate' as TransformHook, + existing: 'existing' as TransformHook, + postCompare: 'postCompare' as TransformHook, +} diff --git a/src/backend/common/infrastructure/config/client/index.ts b/src/backend/common/infrastructure/config/client/index.ts index 3699467c..13d77d5f 100644 --- a/src/backend/common/infrastructure/config/client/index.ts +++ b/src/backend/common/infrastructure/config/client/index.ts @@ -1,4 +1,4 @@ -import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; +import { CommonConfig, CommonData, PlayTransformConfig, RequestRetryOptions } from "../common.js"; /** * Scrobble matching (between new source track and existing client scrobbles) logging options. Used for debugging. @@ -73,6 +73,8 @@ export interface CommonClientOptions extends RequestRetryOptions { * @examples [1] * */ deadLetterRetries?: number + + playTransform?: PlayTransformConfig } export interface CommonClientConfig extends CommonConfig { diff --git a/src/backend/common/infrastructure/config/common.ts b/src/backend/common/infrastructure/config/common.ts index 3caae73d..5718827f 100644 --- a/src/backend/common/infrastructure/config/common.ts +++ b/src/backend/common/infrastructure/config/common.ts @@ -1,4 +1,4 @@ -import { keyOmit } from "../Atomic.js"; +import { keyOmit, PlayTransformHooks, SearchAndReplaceTerm } from "../Atomic.js"; export interface CommonConfig { name?: string @@ -49,3 +49,5 @@ export interface PollingOptions { * */ maxInterval?: number } + +export type PlayTransformConfig = PlayTransformHooks; diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts index 2e9e87a2..f24193b4 100644 --- a/src/backend/common/infrastructure/config/source/index.ts +++ b/src/backend/common/infrastructure/config/source/index.ts @@ -1,4 +1,4 @@ -import { CommonConfig, CommonData, RequestRetryOptions } from "../common.js"; +import { CommonConfig, CommonData, PlayTransformConfig, RequestRetryOptions } from "../common.js"; export interface SourceRetryOptions extends RequestRetryOptions { /** @@ -96,6 +96,8 @@ export interface CommonSourceOptions extends SourceRetryOptions { * * If not specified it defaults to the maximum number of listens the source API supports * */ scrobbleBacklogCount?: number + + playTransform?: PlayTransformConfig } export interface CommonSourceData extends CommonData { diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index 627ea495..e1891eed 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -73,13 +73,13 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i queuedScrobbles: QueuedScrobble[] = []; deadLetterScrobbles: DeadLetterScrobble[] = []; - config: CommonClientConfig; + declare config: CommonClientConfig; notifier: Notifiers; emitter: EventEmitter; constructor(type: any, name: any, config: CommonClientConfig, notifier: Notifiers, emitter: EventEmitter, logger: Logger) { - super(); + super(config); this.type = type; this.name = name; this.identifier = `${capitalize(this.type)} - ${name}`; @@ -89,7 +89,6 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i this.scrobbledPlayObjs = new FixedSizeList(this.MAX_STORED_SCROBBLES); - this.config = config; const { options: { refreshEnabled = true, diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index 11bea18a..d5ed6640 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -46,7 +46,7 @@ export default abstract class AbstractSource extends AbstractComponent implement type: SourceType; identifier: string; - config: SourceConfig; + declare config: SourceConfig; clients: string[]; instantiatedAt: Dayjs; lastActivityAt: Dayjs; @@ -77,7 +77,7 @@ export default abstract class AbstractSource extends AbstractComponent implement protected recentDiscoveredPlays: GroupedFixedPlays = new TupleMap>(); constructor(type: SourceType, name: string, config: SourceConfig, internal: InternalConfig, emitter: EventEmitter) { - super(); + super(config); const {clients = [] } = config; this.type = type; this.name = name; diff --git a/src/backend/tests/component/component.test.ts b/src/backend/tests/component/component.test.ts new file mode 100644 index 00000000..b908324c --- /dev/null +++ b/src/backend/tests/component/component.test.ts @@ -0,0 +1,172 @@ +import { loggerTest } from "@foxxmd/logging"; +import chai, { assert, expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import { after, before, describe, it } from 'mocha'; +import { PlayObject } from "../../../core/Atomic.js"; +import AbstractComponent from "../../common/AbstractComponent.js"; +import { TRANSFORM_HOOK } from "../../common/infrastructure/Atomic.js"; +import { isSearchAndReplace } from "../../utils.js"; +import { asPlays, generatePlay, normalizePlays } from "../utils/PlayTestUtils.js"; + +chai.use(asPromised); + +class TestComponent extends AbstractComponent { + constructor() { + super({}); + } +} + +const component = new TestComponent(); +component.logger = loggerTest; + +describe('Play Transforms', function () { + + beforeEach(function() { + component.config = {}; + component.transformRules = undefined; + }); + + describe('Transform Config Parsing', function() { + + it('Sets transform rules as empty object if config is not present', function() { + component.buildTransformRules(); + expect(component.transformRules).exist; + expect(Object.keys(component.transformRules).length).eq(0); + }); + + it('Converts transform config into real S&P data', function() { + component.config = { + options: { + playTransform: { + preCompare: { + title: ['something'] + } + } + } + } + + component.buildTransformRules(); + + expect(component.transformRules.preCompare).to.exist; + expect(component.transformRules.preCompare.title).to.exist; + expect(Array.isArray(component.transformRules.preCompare.title)).is.true; + expect( isSearchAndReplace(component.transformRules.preCompare.title[0])).is.true + }); + + it('Converts transform config into real S&P data with default being empty string', function() { + component.config = { + options: { + playTransform: { + preCompare: { + title: ['something'] + } + } + } + } + + component.buildTransformRules(); + + expect(component.transformRules.preCompare).to.exist; + expect(component.transformRules.preCompare.title).to.exist; + expect(Array.isArray(component.transformRules.preCompare.title)).is.true; + expect( isSearchAndReplace(component.transformRules.preCompare.title[0])).is.true + expect( component.transformRules.preCompare.title[0].search).is.eq('something'); + expect( component.transformRules.preCompare.title[0].replace).is.eq(''); + }); + + it('Respects transform config when it is already S&P data', function() { + component.config = { + options: { + playTransform: { + preCompare: { + title: [ + { + search: 'nothing', + replace: 'anything' + } + ] + } + } + } + } + + component.buildTransformRules(); + + expect(component.transformRules.preCompare).to.exist; + expect(component.transformRules.preCompare.title).to.exist; + expect(Array.isArray(component.transformRules.preCompare.title)).is.true; + expect( isSearchAndReplace(component.transformRules.preCompare.title[0])).is.true + expect( component.transformRules.preCompare.title[0].search).is.eq('nothing'); + expect( component.transformRules.preCompare.title[0].replace).is.eq('anything'); + }); + }); + + describe('Play Transforming', function() { + + it('Returns original play if no hooks are defined', function () { + component.buildTransformRules(); + + const play = generatePlay(); + const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + expect(JSON.stringify(play)).equal(JSON.stringify(transformed)); + }); + + it('Transforms when hook is present', function () { + component.config = { + options: { + playTransform: { + preCompare: { + title: ['something'] + } + } + } + } + component.buildTransformRules(); + + const play = generatePlay({track: 'My coolsomething track'}); + const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + expect(transformed.data.track).equal('My cool track'); + }); + + it('Transforms consecutively when hook is present with multiple values', function () { + component.config = { + options: { + playTransform: { + preCompare: { + title: ['something', 'cool'] + } + } + } + } + component.buildTransformRules(); + + const play = generatePlay({track: 'My coolsomething track'}); + const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + expect(transformed.data.track).equal('My track'); + }); + + it('Transforms using parsed regex', function () { + component.config = { + options: { + playTransform: { + preCompare: { + title: [ + { + search: '/(cool )(some)(thing)/i', + replace: '$1$3' + } + ] + } + } + } + } + component.buildTransformRules(); + + const play = generatePlay({track: 'My cool something track'}); + const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + expect(transformed.data.track).equal('My cool thing track'); + }); + }); + + +}) diff --git a/src/backend/utils.ts b/src/backend/utils.ts index 0ffd92f2..49945a7e 100644 --- a/src/backend/utils.ts +++ b/src/backend/utils.ts @@ -1,4 +1,5 @@ import { Logger } from '@foxxmd/logging'; +import { SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core"; import backoffStrategies from '@kenyip/backoff-strategies'; import address from "address"; import * as AjvNS from 'ajv'; @@ -22,11 +23,11 @@ import { NO_USER, numberFormatOptions, PlayerStateData, - PlayPlatformId, + PlayPlatformId, PlayTransformParts, ProgressAwarePlayObject, RegExResult, RemoteIdentityParts, - ScrobbleThresholdResult, + ScrobbleThresholdResult, SearchAndReplaceTerm, } from "./common/infrastructure/Atomic.js"; //const { default: Ajv } = AjvNS; @@ -790,3 +791,64 @@ export const joinedUrl = (url: URL, ...paths: string[]): URL => { finalUrl.pathname = joinPath(url.pathname, ...(paths.filter(x => x.trim() !== ''))); return finalUrl; } + +export const configValToSearchReplace = (val: string | undefined | object): SearchAndReplaceRegExp | undefined => { + if (val === undefined || val === null) { + return undefined; + } + if (typeof val === 'string') { + return { + search: val, + replace: '' + } + } + if (isSearchAndReplace(val)) { + return val as SearchAndReplaceRegExp; + } + throw new Error(`Value must be a string or an object containing 'search: string' and 'replace: 'string'. Given: ${val}`); +} + +export const isSearchAndReplace = (val: object): val is SearchAndReplaceRegExp => { + return typeof val === 'object' + && ('search' in val && typeof val.search === 'string') + && ('replace' in val && typeof val.replace === 'string'); +} + +export const configPartsToStrongParts = (val: PlayTransformParts | undefined): PlayTransformParts => { + if (val === undefined) { + return {} + } + const { + title: titleConfig, + artists: artistConfig, + album: albumConfig + } = val; + let title, + artists, + album; + + if (titleConfig !== undefined) { + if (!Array.isArray(titleConfig)) { + throw new Error('title must be an array'); + } + title = titleConfig.map(configValToSearchReplace); + } + if (artistConfig !== undefined) { + if (!Array.isArray(artistConfig)) { + throw new Error('arist must be an array'); + } + artists = artistConfig.map(configValToSearchReplace); + } + if (albumConfig !== undefined) { + if (!Array.isArray(albumConfig)) { + throw new Error('albumConfig must be an array'); + } + album = albumConfig.map(configValToSearchReplace); + } + + return { + title, + artists, + album + } +} From 0e008ecd5c1417076a9df740cffa3592f2d0453f Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 25 Jul 2024 15:03:39 -0400 Subject: [PATCH 11/23] feat: Add album option to play string building function --- src/core/Atomic.ts | 5 +++-- src/core/StringUtils.ts | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/core/Atomic.ts b/src/core/Atomic.ts index 363de34a..7dbc6d8e 100644 --- a/src/core/Atomic.ts +++ b/src/core/Atomic.ts @@ -31,13 +31,14 @@ export interface ClientStatusData { initialized: boolean; } -export type PlayObjectIncludeTypes = 'time' | 'artist' | 'track' | 'timeFromNow' | 'trackId' | 'comment'; -export const recentIncludes: PlayObjectIncludeTypes[] = ['time', 'timeFromNow', 'track', 'artist', 'comment']; +export type PlayObjectIncludeTypes = 'album' | 'time' | 'artist' | 'track' | 'timeFromNow' | 'trackId' | 'comment'; +export const recentIncludes: PlayObjectIncludeTypes[] = ['time', 'timeFromNow', 'track', 'album', 'artist', 'comment']; export interface TrackStringOptions { include?: PlayObjectIncludeTypes[] transformers?: { artists?: (a: string[]) => T | string + album?: (t: string,data: AmbPlayObject, hasExistingParts?: boolean) => T | string track?: (t: string,data: AmbPlayObject, hasExistingParts?: boolean) => T | string time?: (t: Dayjs, i?: ScrobbleTsSOC) => T | string timeFromNow?: (t: Dayjs) => T | string diff --git a/src/core/StringUtils.ts b/src/core/StringUtils.ts index aee5093c..53745809 100644 --- a/src/core/StringUtils.ts +++ b/src/core/StringUtils.ts @@ -33,12 +33,14 @@ export const truncateStringToLength = (length: any, truncStr = '...') => (val: a export const defaultTrackTransformer = (input: any, data: AmbPlayObject, hasExistingParts: boolean = false) => hasExistingParts ? `- ${input}` : input; export const defaultReducer = (acc, curr) => `${acc} ${curr}`; export const defaultArtistFunc = (a: string[]) => a.join(' / '); +export const defaultAlbumFunc = (input: any, data: AmbPlayObject, hasExistingParts: boolean = false) => hasExistingParts ? `--- ${input}` : input; export const defaultTimeFunc = (t: Dayjs | undefined, i?: ScrobbleTsSOC) => t === undefined ? '@ N/A' : `@ ${t.local().format()} ${i === undefined ? '' : (i === SCROBBLE_TS_SOC_START ? '(S)' : '(C)')}`; export const defaultTimeFromNowFunc = (t: Dayjs | undefined) => t === undefined ? undefined : `(${t.local().fromNow()})`; export const defaultCommentFunc = (c: string | undefined) => c === undefined ? undefined : `(${c})`; export const defaultBuildTrackStringTransformers = { artists: defaultArtistFunc, track: defaultTrackTransformer, + album: defaultAlbumFunc, time: defaultTimeFunc, timeFromNow: defaultTimeFromNowFunc, comment: defaultCommentFunc @@ -48,6 +50,7 @@ export const buildTrackString = (playObj: AmbPlayObject, options: Tr include = ['time', 'artist', 'track'], transformers: { artists: artistsFunc = defaultBuildTrackStringTransformers.artists, + album: albumFunc = defaultBuildTrackStringTransformers.album, track: trackFunc = defaultBuildTrackStringTransformers.track, time: timeFunc = defaultBuildTrackStringTransformers.time, timeFromNow = defaultBuildTrackStringTransformers.timeFromNow, @@ -89,6 +92,9 @@ export const buildTrackString = (playObj: AmbPlayObject, options: Tr if (include.includes('track')) { strParts.push(trackFunc(track, playObj, strParts.length > 0)); } + if (include.includes('album')) { + strParts.push(albumFunc(album, playObj, strParts.length > 0)); + } if (include.includes('time')) { strParts.push(timeFunc(pd, usedTsSOC)); } From 9ede4634cb2bc28f2a2d1c64d02bc828ced67d06 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 25 Jul 2024 15:04:37 -0400 Subject: [PATCH 12/23] test: Use faker 9.0RC for more plausible artist/album names in generated data --- package-lock.json | 12 ++++++------ package.json | 2 +- src/backend/tests/utils/PlayTestUtils.ts | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9570a48f..303bfd6a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -79,7 +79,7 @@ "devDependencies": { "@dbus-types/notifications": "^0.0.5", "@eslint/js": "^8.56.0", - "@faker-js/faker": "^8.1.0", + "@faker-js/faker": "^9.0.0-rc.0", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^13.4.0", @@ -1121,9 +1121,9 @@ } }, "node_modules/@faker-js/faker": { - "version": "8.4.1", - "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-8.4.1.tgz", - "integrity": "sha512-XQ3cU+Q8Uqmrbf2e0cIC/QN43sTBSC8KF12u29Mb47tWrt2hAgBXSgpZMj4Ao8Uk0iJcU99QsOCaIL8934obCg==", + "version": "9.0.0-rc.0", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-9.0.0-rc.0.tgz", + "integrity": "sha512-8ouDqdlHqREHizwGk1QiXuL73vtBfRFh0Uh71aJyotCg6QMF9IgyvIsxf/IunoEr3gqpUwAlrhLMwLIvRK1mzg==", "dev": true, "funding": [ { @@ -1132,8 +1132,8 @@ } ], "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0", - "npm": ">=6.14.13" + "node": ">=18.0.0", + "npm": ">=9.0.0" } }, "node_modules/@fortawesome/fontawesome-common-types": { diff --git a/package.json b/package.json index d9d4aae4..425beefe 100644 --- a/package.json +++ b/package.json @@ -114,7 +114,7 @@ "devDependencies": { "@dbus-types/notifications": "^0.0.5", "@eslint/js": "^8.56.0", - "@faker-js/faker": "^8.1.0", + "@faker-js/faker": "^9.0.0-rc.0", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^13.4.0", diff --git a/src/backend/tests/utils/PlayTestUtils.ts b/src/backend/tests/utils/PlayTestUtils.ts index b855f919..cad4545a 100644 --- a/src/backend/tests/utils/PlayTestUtils.ts +++ b/src/backend/tests/utils/PlayTestUtils.ts @@ -81,10 +81,10 @@ export const generatePlay = (data: ObjectPlayData = {}, meta: PlayMeta = {}): Pl return { data: { track: faker.music.songName(), - artists: faker.helpers.multiple(faker.person.fullName, {count: {min: 1, max: 2}}), + artists: faker.helpers.multiple(faker.music.artist, {count: {min: 1, max: 3}}), duration: faker.number.int({min: 30, max: 300}), playDate: dayjs().subtract(faker.number.int({min: 1, max: 800})), - album: faker.music.songName(), + album: faker.music.album(), ...data }, meta: { From b0265001f8d5e1db43e2e9071f20e0d35ff78219 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 25 Jul 2024 15:12:39 -0400 Subject: [PATCH 13/23] feat: Improve logging for play transforms * Use a proper child logger and error messages on catch * Log full play string before => after instead of just parts --- src/backend/common/AbstractComponent.ts | 219 +++++++++++------- .../infrastructure/config/client/index.ts | 4 +- .../common/infrastructure/config/common.ts | 2 + .../infrastructure/config/source/index.ts | 4 +- src/backend/tests/component/component.test.ts | 5 +- 5 files changed, 139 insertions(+), 95 deletions(-) diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 3a45b630..7071462b 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -1,7 +1,8 @@ -import { Logger } from "@foxxmd/logging"; +import { childLogger, Logger } from "@foxxmd/logging"; import { searchAndReplace, SearchAndReplaceRegExp } from "@foxxmd/regex-buddy-core"; import { compare } from "compare-versions"; -import { PlayObject } from "../../core/Atomic.js"; +import { ObjectPlayData, PlayData, PlayObject } from "../../core/Atomic.js"; +import { buildTrackString } from "../../core/StringUtils.js"; import { configPartsToStrongParts, configValToSearchReplace } from "../utils.js"; import { hasNodeNetworkException } from "./errors/NodeErrors.js"; import { hasUpstreamError } from "./errors/UpstreamError.js"; @@ -234,114 +235,156 @@ export default abstract class AbstractComponent { return; } - public transformPlay = (play: PlayObject, hookType: TransformHook, log: boolean = true) => { - let hook: PlayTransformParts | undefined; - - switch (hookType) { - case TRANSFORM_HOOK.preCompare: - hook = this.transformRules.preCompare; - break; - case TRANSFORM_HOOK.candidate: - hook = this.transformRules.compare?.candidate; - break; - case TRANSFORM_HOOK.existing: - hook = this.transformRules.compare?.candidate; - break; - case TRANSFORM_HOOK.postCompare: - hook = this.transformRules.postCompare; - break; - } + public transformPlay = (play: PlayObject, hookType: TransformHook, log?: boolean) => { - if (hook === undefined) { - return play; - } + let logger: Logger; + const labels = ['Play Transform', hookType]; + const getLogger = () => logger !== undefined ? logger : childLogger(this.logger, labels); - const { - data: { - track, - artists, - albumArtists, - album - } = {} - } = play; + try { + let hook: PlayTransformParts | undefined; + + switch (hookType) { + case TRANSFORM_HOOK.preCompare: + hook = this.transformRules.preCompare; + break; + case TRANSFORM_HOOK.candidate: + hook = this.transformRules.compare?.candidate; + break; + case TRANSFORM_HOOK.existing: + hook = this.transformRules.compare?.candidate; + break; + case TRANSFORM_HOOK.postCompare: + hook = this.transformRules.postCompare; + break; + } - const transformedPlay = { - ...play - } + if (hook === undefined) { + return play; + } - const results: [string, string, string][] = []; + const { + data: { + track, + artists, + albumArtists, + album + } = {} + } = play; - if (hook.title !== undefined && track !== undefined) { - try { - const t = searchAndReplace(track, hook.title); - if (t !== track) { - results.push(['title', track, t]); - transformedPlay.data.track = t; - } - } catch (e) { - this.logger.warn(`Failed to transform title during ${hookType} transform: ${track}`, {cause: e}); - } - } + const transformedPlayData: Partial = {}; + + //const results: [string, string, string][] = []; + let isTransformed = false; - if (hook.artists !== undefined && artists !== undefined && artists.length > 0) { - const transformedArtists: string[] = []; - let anyTransformed = false; - for (const artist of artists) { + if (hook.title !== undefined && track !== undefined) { try { - const t = searchAndReplace(artist, hook.artists); - if (t !== artist) { - anyTransformed = true; + const t = searchAndReplace(track, hook.title); + if (t !== track) { + //results.push(['title', track, t]); + transformedPlayData.track = t; + //transformedPlay.data.track = t; + isTransformed = true; } - transformedArtists.push(t); } catch (e) { - this.logger.warn(`Failed to transform artist during ${hookType} transform: ${artist}`, {cause: e}); - transformedArtists.push(artist); + getLogger().warn(new Error(`Failed to transform title: ${track}`, {cause: e})); } } - transformedPlay.data.artists = transformedArtists; - if (anyTransformed) { - results.push(['artists', artists.join(' / '), transformedArtists.join(' / ')]); + + if (hook.artists !== undefined && artists !== undefined && artists.length > 0) { + const transformedArtists: string[] = []; + let anyArtistTransformed = false; + for (const artist of artists) { + try { + const t = searchAndReplace(artist, hook.artists); + if (t !== artist) { + anyArtistTransformed = true; + isTransformed = true; + } + transformedArtists.push(t); + } catch (e) { + getLogger().warn(new Error(`Failed to transform artist: ${artist}`, {cause: e})); + transformedArtists.push(artist); + } + } + if(anyArtistTransformed) { + transformedPlayData.artists = transformedArtists; + } + //transformedPlay.data.artists = transformedArtists; + // if (anyTransformed) { + // //results.push(['artists', artists.join(' / '), transformedArtists.join(' / ')]); + // } } - } - if (hook.artists !== undefined && albumArtists !== undefined && albumArtists.length > 0) { - const transformedArtists: string[] = []; - let anyTransformed = false; - for (const artist of albumArtists) { + if (hook.artists !== undefined && albumArtists !== undefined && albumArtists.length > 0) { + const transformedArtists: string[] = []; + let anyArtistTransformed = false; + for (const artist of albumArtists) { + try { + const t = searchAndReplace(artist, hook.artists); + if (t !== artist) { + anyArtistTransformed = true; + isTransformed = true; + } + transformedArtists.push(t); + } catch (e) { + getLogger().warn(new Error(`Failed to transform albumArtist: ${artist}`, {cause: e})); + transformedArtists.push(artist); + } + } + if(anyArtistTransformed) { + transformedPlayData.albumArtists = transformedArtists; + } + // transformedPlay.data.albumArtists = transformedArtists; + // if (anyTransformed) { + // results.push(['albumArtists', artists.join(' / '), transformedArtists.join(' / ')]); + // } + } + + if (hook.album !== undefined && album !== undefined) { try { - const t = searchAndReplace(artist, hook.artists); - if (t !== artist) { - anyTransformed = true; + const t = searchAndReplace(album, hook.album); + if (t !== album) { + isTransformed = true; + transformedPlayData.album = t; + // results.push(['album', album, t]); + // transformedPlay.data.album = t; } - transformedArtists.push(t); } catch (e) { - this.logger.warn(`Failed to transform albumArtist during ${hookType} transform: ${artist}`, {cause: e}); - transformedArtists.push(artist); + getLogger().warn(new Error(`Failed to transform album: ${album}`, {cause: e})); } } - transformedPlay.data.albumArtists = transformedArtists; - if (anyTransformed) { - results.push(['albumArtists', artists.join(' / '), transformedArtists.join(' / ')]); - } - } - if (hook.album !== undefined && album !== undefined) { - try { - const t = searchAndReplace(album, hook.album); - if (t !== album) { - results.push(['album', album, t]); - transformedPlay.data.album = t; + if(isTransformed) { + + const transformedPlay = { + ...play, + data: { + ...play.data, + ...transformedPlayData + } } - } catch (e) { - this.logger.warn(`Failed to transform album during ${hookType} transform: ${album}`, {cause: e}); + + const shouldLog = log ?? this.config.options?.playTransform?.log ?? true; + if(shouldLog) { + this.logger.debug({labels}, `Play transformed by ${hookType}: +Original : ${buildTrackString(play, {include: ['artist', 'track', 'album']})} +Transformed : ${buildTrackString(transformedPlay, {include: ['artist', 'track', 'album']})} +`); + } + + return transformedPlay; } - } - if(results.length > 0) { - this.logger.debug(`Play transformed by ${hookType}: - ${results.map(x => `${x[0]}: ${x[1]} => ${x[2]}`).join('\n')}`); - } +// if (results.length > 0 && log) { +// this.logger.debug({labels}, `Play transformed by ${hookType}: +// ${results.map(x => `${x[0]}: ${x[1]} => ${x[2]}`).join('\n')}`); +// } - return transformedPlay; + return play; + } catch (e) { + getLogger().warn(new Error(`Unexpected error occurred, returning original play.`, {cause: e})); + return play; + } } } diff --git a/src/backend/common/infrastructure/config/client/index.ts b/src/backend/common/infrastructure/config/client/index.ts index 13d77d5f..bffcf589 100644 --- a/src/backend/common/infrastructure/config/client/index.ts +++ b/src/backend/common/infrastructure/config/client/index.ts @@ -1,4 +1,4 @@ -import { CommonConfig, CommonData, PlayTransformConfig, RequestRetryOptions } from "../common.js"; +import { CommonConfig, CommonData, PlayTransformConfig, PlayTransformOptions, RequestRetryOptions } from "../common.js"; /** * Scrobble matching (between new source track and existing client scrobbles) logging options. Used for debugging. @@ -74,7 +74,7 @@ export interface CommonClientOptions extends RequestRetryOptions { * */ deadLetterRetries?: number - playTransform?: PlayTransformConfig + playTransform?: PlayTransformOptions } export interface CommonClientConfig extends CommonConfig { diff --git a/src/backend/common/infrastructure/config/common.ts b/src/backend/common/infrastructure/config/common.ts index 5718827f..555841a2 100644 --- a/src/backend/common/infrastructure/config/common.ts +++ b/src/backend/common/infrastructure/config/common.ts @@ -51,3 +51,5 @@ export interface PollingOptions { } export type PlayTransformConfig = PlayTransformHooks; + +export type PlayTransformOptions = PlayTransformConfig & { log?: boolean } diff --git a/src/backend/common/infrastructure/config/source/index.ts b/src/backend/common/infrastructure/config/source/index.ts index f24193b4..1ee153cd 100644 --- a/src/backend/common/infrastructure/config/source/index.ts +++ b/src/backend/common/infrastructure/config/source/index.ts @@ -1,4 +1,4 @@ -import { CommonConfig, CommonData, PlayTransformConfig, RequestRetryOptions } from "../common.js"; +import { CommonConfig, CommonData, PlayTransformConfig, PlayTransformOptions, RequestRetryOptions } from "../common.js"; export interface SourceRetryOptions extends RequestRetryOptions { /** @@ -97,7 +97,7 @@ export interface CommonSourceOptions extends SourceRetryOptions { * */ scrobbleBacklogCount?: number - playTransform?: PlayTransformConfig + playTransform?: PlayTransformOptions } export interface CommonSourceData extends CommonData { diff --git a/src/backend/tests/component/component.test.ts b/src/backend/tests/component/component.test.ts index b908324c..13f2be4f 100644 --- a/src/backend/tests/component/component.test.ts +++ b/src/backend/tests/component/component.test.ts @@ -1,8 +1,7 @@ -import { loggerTest } from "@foxxmd/logging"; +import { loggerTest, loggerDebug, childLogger } from "@foxxmd/logging"; import chai, { assert, expect } from 'chai'; import asPromised from 'chai-as-promised'; import { after, before, describe, it } from 'mocha'; -import { PlayObject } from "../../../core/Atomic.js"; import AbstractComponent from "../../common/AbstractComponent.js"; import { TRANSFORM_HOOK } from "../../common/infrastructure/Atomic.js"; import { isSearchAndReplace } from "../../utils.js"; @@ -17,7 +16,7 @@ class TestComponent extends AbstractComponent { } const component = new TestComponent(); -component.logger = loggerTest; +component.logger = childLogger(loggerTest, 'App'); describe('Play Transforms', function () { From 32220d7509e64c51917ed5de75bbc2f82a0f7abc Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 25 Jul 2024 16:16:58 -0400 Subject: [PATCH 14/23] feat: Implement play transforms in source/client logic --- src/backend/common/AbstractComponent.ts | 2 +- .../scrobblers/AbstractScrobbleClient.ts | 51 +++--- src/backend/sources/AbstractSource.ts | 14 +- .../tests/scrobbler/scrobblers.test.ts | 162 ++++++++++++++++-- src/backend/tests/source/TestSource.ts | 8 + src/backend/tests/source/source.test.ts | 127 ++++++++++++++ 6 files changed, 325 insertions(+), 39 deletions(-) create mode 100644 src/backend/tests/source/TestSource.ts create mode 100644 src/backend/tests/source/source.test.ts diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index 7071462b..b50db2fb 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -252,7 +252,7 @@ export default abstract class AbstractComponent { hook = this.transformRules.compare?.candidate; break; case TRANSFORM_HOOK.existing: - hook = this.transformRules.compare?.candidate; + hook = this.transformRules.compare?.existing; break; case TRANSFORM_HOOK.postCompare: hook = this.transformRules.postCompare; diff --git a/src/backend/scrobblers/AbstractScrobbleClient.ts b/src/backend/scrobblers/AbstractScrobbleClient.ts index e1891eed..1696c0e3 100644 --- a/src/backend/scrobblers/AbstractScrobbleClient.ts +++ b/src/backend/scrobblers/AbstractScrobbleClient.ts @@ -3,6 +3,7 @@ import dayjs, { Dayjs } from "dayjs"; import EventEmitter from "events"; import { FixedSizeList } from 'fixed-size-list'; import { nanoid } from "nanoid"; +import { Simulate } from "react-dom/test-utils"; import { DeadLetterScrobble, PlayObject, @@ -23,7 +24,7 @@ import { FormatPlayObjectOptions, ScrobbledPlayObject, TIME_WEIGHT, - TITLE_WEIGHT, + TITLE_WEIGHT, TRANSFORM_HOOK, } from "../common/infrastructure/Atomic.js"; import { CommonClientConfig } from "../common/infrastructure/config/client/index.js"; import { Notifiers } from "../notifier/Notifiers.js"; @@ -244,17 +245,13 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i getScrobbledPlays = () => this.scrobbledPlayObjs.data.map(x => x.scrobble) - findExistingSubmittedPlayObj = (playObj: PlayObject): ([undefined, undefined] | [ScrobbledPlayObject, ScrobbledPlayObject[]]) => { - const { - data: { - playDate - } = {}, - meta: { - source, - } = {} - } = playObj; + findExistingSubmittedPlayObj = (playObjPre: PlayObject): ([undefined, undefined] | [ScrobbledPlayObject, ScrobbledPlayObject[]]) => { - const dtInvariantMatches = this.scrobbledPlayObjs.data.filter(x => playObjDataMatch(playObj, x.play)); + const playObj = this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); + + const dtInvariantMatches = this.scrobbledPlayObjs.data + .map(x => ({...x, play: this.transformPlay(x.play, TRANSFORM_HOOK.existing)})) + .filter(x => playObjDataMatch(playObj, x.play)); if (dtInvariantMatches.length === 0) { return [undefined, []]; @@ -291,7 +288,10 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i return [Math.min(compareScrobbleArtists(existing, candidate)/100, 1), wholeMatches] } - existingScrobble = async (playObj: PlayObject) => { + existingScrobble = async (playObjPre: PlayObject) => { + + const playObj = this.transformPlay(playObjPre, TRANSFORM_HOOK.candidate); + const tr = truncateStringToLength(27); const scoreTrackOpts: TrackStringOptions = {include: ['track', 'artist', 'time'], transformers: {track: (t: any, data, existing) => `${existing ? '- ': ''}${tr(t)}`}}; @@ -307,7 +307,7 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i let closestMatch: {score: number, breakdowns: string[], confidence: string, scrobble?: PlayObject} = {score: 0, breakdowns: [], confidence: 'No existing scrobble matched with a score higher than 0'}; // then check if we have already recorded this - const [existingExactSubmitted, existingDataSubmitted = []] = this.findExistingSubmittedPlayObj(playObj); + const [existingExactSubmitted, existingDataSubmitted = []] = this.findExistingSubmittedPlayObj(playObjPre); // if we have an submitted play with matching data and play date then we can just return the response from the original scrobble if (existingExactSubmitted !== undefined) { @@ -338,7 +338,9 @@ export default abstract class AbstractScrobbleClient extends AbstractComponent i // in which case we can check the scrobble api response against recent scrobbles (also from api) for a more accurate comparison const referenceApiScrobbleResponse = existingDataSubmitted.length > 0 ? existingDataSubmitted[0].scrobble : undefined; - existingScrobble = this.recentScrobbles.find((x) => { + existingScrobble = this.recentScrobbles.find((xPre) => { + + const x = this.transformPlay(xPre, TRANSFORM_HOOK.existing); //const referenceMatch = referenceApiScrobbleResponse !== undefined && playObjDataMatch(x, referenceApiScrobbleResponse); @@ -582,14 +584,15 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); const currQueuedPlay = this.queuedScrobbles.shift(); const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(currQueuedPlay.play); if (timeFrameValid && !(await this.alreadyScrobbled(currQueuedPlay.play))) { + const transformedScrobble = this.transformPlay(currQueuedPlay.play, TRANSFORM_HOOK.postCompare); try { - const scrobbledPlay = await this.scrobble(currQueuedPlay.play); - this.emitEvent('scrobble', {play: currQueuedPlay.play}); - this.addScrobbledTrack(currQueuedPlay.play, scrobbledPlay); + const scrobbledPlay = await this.scrobble(transformedScrobble); + this.emitEvent('scrobble', {play: transformedScrobble}); + this.addScrobbledTrack(transformedScrobble, scrobbledPlay); } catch (e) { if (e instanceof UpstreamError && e.showStopper === false) { this.addDeadLetterScrobble(currQueuedPlay, e); - this.logger.warn(new Error(`Could not scrobble ${buildTrackString(currQueuedPlay.play)} from Source '${currQueuedPlay.source}' but error was not show stopping. Adding scrobble to Dead Letter Queue and will retry on next heartbeat.`, {cause: e})); + this.logger.warn(new Error(`Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${currQueuedPlay.source}' but error was not show stopping. Adding scrobble to Dead Letter Queue and will retry on next heartbeat.`, {cause: e})); } else { this.queuedScrobbles.unshift(currQueuedPlay); throw new Error('Error occurred while trying to scrobble', {cause: e}); @@ -664,15 +667,16 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); } const [timeFrameValid, timeFrameValidLog] = this.timeFrameIsValid(deadScrobble.play); if (timeFrameValid && !(await this.alreadyScrobbled(deadScrobble.play))) { + const transformedScrobble = this.transformPlay(deadScrobble.play, TRANSFORM_HOOK.postCompare); try { - const scrobbledPlay = await this.scrobble(deadScrobble.play); - this.emitEvent('scrobble', {play: deadScrobble.play}); - this.addScrobbledTrack(deadScrobble.play, scrobbledPlay); + const scrobbledPlay = await this.scrobble(transformedScrobble); + this.emitEvent('scrobble', {play: transformedScrobble}); + this.addScrobbledTrack(transformedScrobble, scrobbledPlay); } catch (e) { deadScrobble.retries++; deadScrobble.error = messageWithCauses(e); deadScrobble.lastRetry = dayjs(); - this.logger.error(new Error(`Could not scrobble ${buildTrackString(deadScrobble.play)} from Source '${deadScrobble.source}' due to error`, {cause: e})); + this.logger.error(new Error(`Could not scrobble ${buildTrackString(transformedScrobble)} from Source '${deadScrobble.source}' due to error`, {cause: e})); this.deadLetterScrobbles[deadScrobbleIndex] = deadScrobble; return [false, deadScrobble]; } finally { @@ -712,7 +716,8 @@ ${closestMatch.breakdowns.join('\n')}`, {leaf: ['Dupe Check']}); queueScrobble = (data: PlayObject | PlayObject[], source: string) => { const plays = Array.isArray(data) ? data : [data]; for(const p of plays) { - const queuedPlay = {id: nanoid(), source, play: p} + const transformedPlay = this.transformPlay(p, TRANSFORM_HOOK.preCompare); + const queuedPlay = {id: nanoid(), source, play: transformedPlay} this.emitEvent('scrobbleQueued', {queuedPlay: queuedPlay}); this.queuedScrobbles.push(queuedPlay); } diff --git a/src/backend/sources/AbstractSource.ts b/src/backend/sources/AbstractSource.ts index d5ed6640..d2f6123b 100644 --- a/src/backend/sources/AbstractSource.ts +++ b/src/backend/sources/AbstractSource.ts @@ -18,7 +18,7 @@ import { PlayUserId, ProgressAwarePlayObject, SINGLE_USER_PLATFORM_ID, - SourceType, + SourceType, TRANSFORM_HOOK, } from "../common/infrastructure/Atomic.js"; import { SourceConfig } from "../common/infrastructure/config/source/sources.js"; import TupleMap from "../common/TupleMap.js"; @@ -147,8 +147,12 @@ export default abstract class AbstractSource extends AbstractComponent implement } }); } + const candidate = this.transformPlay(play, TRANSFORM_HOOK.candidate); for(const list of lists) { - const existing = list.find(x => playObjDataMatch(x, play) && temporalAccuracyIsAtLeast(TA_CLOSE, comparePlayTemporally(x, play).match)); + const existing = list.find(x => { + const e = this.transformPlay(x, TRANSFORM_HOOK.existing); + return playObjDataMatch(e, candidate) && temporalAccuracyIsAtLeast(TA_CLOSE, comparePlayTemporally(e, candidate).match) + }); if(existing) { return existing; } @@ -166,7 +170,9 @@ export default abstract class AbstractSource extends AbstractComponent implement discover = (plays: PlayObject[], options: { checkAll?: boolean, [key: string]: any } = {}): PlayObject[] => { const newDiscoveredPlays: PlayObject[] = []; - for(const play of plays) { + const transformedPlayed = plays.map(x => this.transformPlay(x, TRANSFORM_HOOK.preCompare)); + + for(const play of transformedPlayed) { if(!this.alreadyDiscovered(play, options)) { this.addPlayToDiscovered(play); newDiscoveredPlays.push(play); @@ -184,7 +190,7 @@ export default abstract class AbstractSource extends AbstractComponent implement if(newDiscoveredPlays.length > 0) { newDiscoveredPlays.sort(sortByOldestPlayDate); this.emitter.emit('discoveredToScrobble', { - data: newDiscoveredPlays, + data: newDiscoveredPlays.map(x => this.transformPlay(x, TRANSFORM_HOOK.postCompare)), options: { ...options, checkTime: newDiscoveredPlays[newDiscoveredPlays.length-1].data.playDate.add(2, 'second'), diff --git a/src/backend/tests/scrobbler/scrobblers.test.ts b/src/backend/tests/scrobbler/scrobblers.test.ts index 6749e080..b2161921 100644 --- a/src/backend/tests/scrobbler/scrobblers.test.ts +++ b/src/backend/tests/scrobbler/scrobblers.test.ts @@ -1,6 +1,7 @@ -import chai, { assert } from 'chai'; +import chai, { assert, expect } from 'chai'; import asPromised from 'chai-as-promised'; import clone from 'clone'; +import { source } from "common-tags"; import dayjs from "dayjs"; import { after, before, describe, it } from 'mocha'; import { http, HttpResponse } from 'msw'; @@ -26,15 +27,20 @@ const normalizedWithMixedDur = normalizePlays(mixedDurPlays, {initialDate: first const normalizedWithMixedDurOlder = normalizePlays(mixedDurPlays, {initialDate: olderFirstPlayDate}); -const testScrobbler = new TestScrobbler(); -testScrobbler.verboseOptions = { - match: { - onMatch: true, - onNoMatch: true, - confidenceBreakdown: true - } -}; -testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); +const generateTestScrobbler = () => { + const testScrobbler = new TestScrobbler(); + testScrobbler.verboseOptions = { + match: { + onMatch: true, + onNoMatch: true, + confidenceBreakdown: true + } + }; + testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); + return testScrobbler; +} + +let testScrobbler: TestScrobbler = generateTestScrobbler() describe('Networking', function () { @@ -94,6 +100,10 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu describe('When scrobble is unique', function () { + beforeEach(function() { + testScrobbler = generateTestScrobbler(); + }); + it('It is not detected as duplicate when play date is newer than most recent', async function () { testScrobbler.recentScrobbles = normalizedWithMixedDur; @@ -137,6 +147,10 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu describe('When scrobble track/artist/album matches existing but is a new scrobble', function () { + beforeEach(function() { + testScrobbler = generateTestScrobbler(); + }); + it('Is not detected as duplicate when artist is same, time is similar, but track is different', async function () { testScrobbler.recentScrobbles = normalizedWithMixedDur; @@ -192,6 +206,11 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu }); describe('When existing has duration', function () { + + beforeEach(function() { + testScrobbler = generateTestScrobbler(); + }); + it('A track with continuity to the previous track is not detected as a duplicate', async function () { testScrobbler.recentScrobbles = normalizedWithDur; @@ -222,6 +241,10 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu describe('When scrobble is a duplicate (title/artists/album)', function () { + beforeEach(function() { + testScrobbler = generateTestScrobbler(); + }); + it('Is detected as duplicate when an exact match', async function () { testScrobbler.recentScrobbles = normalizedWithMixedDur; assert.isTrue(await testScrobbler.alreadyScrobbled(normalizedWithMixedDur[normalizedWithMixedDur.length - 1])); @@ -355,6 +378,10 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu describe('When at least one play has duration', function () { + beforeEach(function() { + testScrobbler = generateTestScrobbler(); + }); + it('Is detected as duplicate when play date is close to the end of an existing scrobble', async function () { testScrobbler.recentScrobbles = normalizedWithDur; @@ -381,11 +408,17 @@ describe('Detects duplicate and unique scrobbles from client recent history', fu describe('Detects duplicate and unique scrobbles using actively tracked scrobbles', function() { - before(function () { + beforeEach(function() { + testScrobbler = generateTestScrobbler(); testScrobbler.recentScrobbles = normalizedWithMixedDur; testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); }); + // before(function () { + // testScrobbler.recentScrobbles = normalizedWithMixedDur; + // testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); + // }); + it('Detects a unique play', async function() { const newScrobble = generatePlay({ playDate: normalizedWithMixedDur[normalizedWithMixedDur.length - 3].data.playDate.add(3, 'seconds') @@ -430,6 +463,7 @@ describe('Detects when upstream scrobbles should be refreshed', function() { const normalizedClose = normalizePlays(withDurPlays, {initialDate: dayjs().subtract(100, 'seconds')}); beforeEach(function () { + testScrobbler = generateTestScrobbler(); testScrobbler.recentScrobbles = normalizedWithMixedDur; testScrobbler.newestScrobbleTime = normalizedWithMixedDur[0].data.playDate; testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); @@ -484,9 +518,115 @@ describe('Detects when upstream scrobbles should be refreshed', function() { }); }); +describe('Scrobble client uses transform plays correctly', function() { + + beforeEach(async function() { + testScrobbler = generateTestScrobbler(); + await testScrobbler.initialize(); + testScrobbler.recentScrobbles = normalizedWithMixedDur; + testScrobbler.scrobbleSleep = 500; + testScrobbler.scrobbleDelay = 0; + testScrobbler.lastScrobbleCheck = dayjs().subtract(60, 'seconds'); + testScrobbler.config.options = {}; + //testScrobbler.initScrobbleMonitoring().catch(console.error); + }); + + it('Transforms play before queue when preCompare is present', async function() { + testScrobbler.config.options = { + playTransform: { + preCompare: { + title: [ + 'cool' + ] + } + } + } + testScrobbler.buildTransformRules(); + const newScrobble = generatePlay({ + track: 'my cool track' + }); + testScrobbler.queueScrobble(newScrobble, 'test'); + expect(testScrobbler.queuedScrobbles[0].play.data.track).is.eq('my track'); + }); + + it('Transforms play on scrobble when postCompare is present', async function() { + testScrobbler.config.options = { + playTransform: { + postCompare: { + title: [ + 'cool' + ] + } + } + } + testScrobbler.buildTransformRules(); + const newScrobble = generatePlay({ + track: 'my cool track' + }); + testScrobbler.queueScrobble(newScrobble, 'test'); + expect(testScrobbler.queuedScrobbles[0].play.data.track).is.eq('my cool track'); + testScrobbler.scrobbleSleep = 100; + testScrobbler.initScrobbleMonitoring().catch(console.error); + + const e = (await pEvent(testScrobbler.emitter, 'scrobble')) as {data: {play: PlayObject }}; + expect(e.data.play.data.track).is.eq('my track'); + }); + + it('Transforms candidate play on comparison', async function() { + testScrobbler.config.options = { + playTransform: { + compare: { + candidate: { + title: [ + 'hugely cool and very different track' + ] + } + } + } + } + const newScrobble = generatePlay({ + track: 'my hugely cool and very different track title' + }); + + testScrobbler.recentScrobbles = normalizePlays([newScrobble, ...withDurPlays], {initialDate: firstPlayDate}); + testScrobbler.buildTransformRules(); + + expect(await testScrobbler.alreadyScrobbled(newScrobble)).is.false; + }); + + it('Transforms existing play on comparison', async function() { + testScrobbler.config.options = { + playTransform: { + compare: { + existing: { + title: [ + 'hugely cool and very different track' + ] + } + } + } + } + const newScrobble = generatePlay({ + track: 'my hugely cool and very different track title' + }); + + testScrobbler.recentScrobbles = normalizePlays([newScrobble, ...withDurPlays], {initialDate: firstPlayDate}); + testScrobbler.buildTransformRules(); + + expect(await testScrobbler.alreadyScrobbled(newScrobble)).is.false; + }); + + afterEach(async function () { + this.timeout(3500); + await testScrobbler.tryStopScrobbling() + }); + +}); + describe('Manages scrobble queue', function() { before(async function() { + testScrobbler = generateTestScrobbler(); await testScrobbler.initialize(); testScrobbler.recentScrobbles = normalizedWithMixedDur; testScrobbler.scrobbleSleep = 500; diff --git a/src/backend/tests/source/TestSource.ts b/src/backend/tests/source/TestSource.ts new file mode 100644 index 00000000..3631b2d0 --- /dev/null +++ b/src/backend/tests/source/TestSource.ts @@ -0,0 +1,8 @@ +import { PlayObject } from "../../../core/Atomic.js"; +import AbstractSource from "../../sources/AbstractSource.js"; + +export class TestSource extends AbstractSource { + handle(plays: PlayObject[]) { + this.scrobble(plays); + } +} diff --git a/src/backend/tests/source/source.test.ts b/src/backend/tests/source/source.test.ts new file mode 100644 index 00000000..8de7b069 --- /dev/null +++ b/src/backend/tests/source/source.test.ts @@ -0,0 +1,127 @@ +import { loggerTest, loggerDebug } from "@foxxmd/logging"; +import chai, { assert, expect } from 'chai'; +import asPromised from 'chai-as-promised'; +import EventEmitter from "events"; +import { after, before, describe, it } from 'mocha'; +import pEvent from "p-event"; +import { PlayObject } from "../../../core/Atomic.js"; +import { generatePlay } from "../utils/PlayTestUtils.js"; +import { TestSource } from "./TestSource.js"; + +chai.use(asPromised); + + +const emitter = new EventEmitter(); +const generateSource = () => { + return new TestSource('spotify', 'test', {}, {localUrl: new URL('https://example.com'), configDir: 'fake', logger: loggerTest}, emitter); +} +let source: TestSource = generateSource(); + +describe('Sources use transform plays correctly', function () { + + beforeEach(function() { + source = generateSource(); + }); + + it('Transforms play on preCompare', function() { + source.config.options = { + playTransform: { + preCompare: { + title: [ + { + search: 'cool', + replace: 'fun' + } + ] + } + } + }; + source.buildTransformRules(); + const newScrobble = generatePlay({ + track: 'my cool track' + }); + const discovered = source.discover([newScrobble]) + expect(discovered.length).eq(1); + expect(discovered[0].data.track).is.eq('my fun track'); + }); + + it('Transforms play on postCompare', async function() { + source.config.options = { + playTransform: { + postCompare: { + title: [ + { + search: 'cool', + replace: 'fun' + } + ] + } + } + }; + source.buildTransformRules(); + const newScrobble = generatePlay({ + track: 'my cool track' + }); + const discovered = source.discover([newScrobble]) + expect(discovered.length).eq(1); + expect(discovered[0].data.track).is.eq('my cool track'); + + const pAwaiter = pEvent(source.emitter, 'discoveredToScrobble') as Promise<{data: [PlayObject] }>; + source.handle(discovered); + const e = await pAwaiter; + expect(e.data.length).is.eq(1); + expect(e.data[0].data.track).is.eq('my fun track'); + }); + + it('Transforms play existing comparison', function() { + source.config.options = { + playTransform: { + compare: { + existing: { + title: [ + { + search: 'hugely cool and very different track', + replace: 'fun' + } + ] + } + } + } + }; + source.buildTransformRules(); + const newScrobble = generatePlay({ + track: 'my hugely cool and very different track title', + }); + const discovered = source.discover([newScrobble]) + expect(discovered.length).eq(1); + expect(discovered[0].data.track).is.eq('my hugely cool and very different track title'); + + expect(source.discover([newScrobble]).length).is.eq(1); + }); + + it('Transforms play candidate comparison', function() { + source.config.options = { + playTransform: { + compare: { + candidate: { + title: [ + { + search: 'hugely cool and very different track', + replace: 'fun' + } + ] + } + } + } + }; + source.buildTransformRules(); + const newScrobble = generatePlay({ + track: 'my hugely cool and very different track title', + }); + const discovered = source.discover([newScrobble]) + expect(discovered.length).eq(1); + expect(discovered[0].data.track).is.eq('my hugely cool and very different track title'); + + expect(source.discover([newScrobble]).length).is.eq(1); + }); +}) From 5c460ad747250279770cbf3a7d00ddc11b30d0df Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 25 Jul 2024 17:28:49 -0400 Subject: [PATCH 15/23] docs: Add Play Transform docs --- docsite/docs/configuration/transforms.mdx | 217 ++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 docsite/docs/configuration/transforms.mdx diff --git a/docsite/docs/configuration/transforms.mdx b/docsite/docs/configuration/transforms.mdx new file mode 100644 index 00000000..5699e9be --- /dev/null +++ b/docsite/docs/configuration/transforms.mdx @@ -0,0 +1,217 @@ +--- +sidebar_position: 4 +title: Scrobble Modification +toc_max_heading_level: 4 +--- + +Multi-scrobbler configs support the ability to modify scrobble data in an automated fashion by matching and replacing strings in **title, artists, and album** at many different times in multi-scrobbler's lifecycle. + +### Why? + +You may need to "clean up" data from a Source or before sending to a scrobble Client due to any number of reasons: + +* ID3 tags in your music collection are dirty or have repeating garbage IE `[YourMusicSource.com] My Artist - My Title` +* A Source's service often incorrectly adds data to some field IE `My Artist - My Title (Album Version)` when the title should just be `My Title` +* An Artist you listen to often is spelled different between a Source and a Client which causes duplicate scrobbles + +In any scenario where a repeating pattern can be found in the data it would be nice to be able to fix it before the data gets downstream or to help prevent duplicate scrobbling. Multi-scrobbler can help you do this. + +## Overview + +### Journey of a Scrobble + +First, let's recap the lifecycle of a scrobble in multi-scrobbler: + +**Sources** are the beginning of the journey for a **Play** (song you've listened to long enough to scrobblable) + +* A Source finds a new valid **Play** +* The Source **compares** this new Play to all the other Plays it has already seen, if the Play is unique (title/artist/album/listened datetime) then... +* The Source **discovers** the Play, adds it to Plays it has seen already, and broadcasts the Play should be scrobbled to all Clients + +Scrobble **Clients** listen for discovered Plays from Sources, then... + +* A Client receives a **Play** from a Source +* The Client **compares** this Play to all the other scrobbles it has already seen, if the Play is unique (title/artist/album/listened datetime) then... +* The Client **scrobbles** the Play downstream to the scrobble service and adds it as a Scrobble it has seen already + +### Lifecyle Hooks + +You'll notice there is a pattern above that looks like this: + +* **Before** data is compared +* Data is **compared** +* **After** data is compared + +These points, during both Source and Client processes, are when you can hook into the scrobble lifecycle and modify it. + +#### TLDR + +In more concrete terms this is the structure of hooks within a configuration (can be used in any **Source** or **Client**): + +```json5 title="lastfm.json" {10-14} +[ + { + "name": "myLastFm", + "enable": true, + "configureAs": "source", + "data": { + // ... + }, + "options": { + "playTransform": { + "preCompare": {/* ... */}, + "compare": {/* ... */}, + "postCompare": {/* ... */} + } + } + } +] +``` + +##### Hook + +For **Sources**: + +* `preCompare` - modify Play data immediately when received +* `compare` - modify Play data when it is being compared to see if Play was already discovered +* `postCompare` - modify Play data before sending to scrobble **Clients** + +For **Clients**: + +* `preCompare` - modify Play data immediately when received +* `compare` - modify Play data when it is being compared to see if it was already scrobbled +* `postCompare` - modify Play data before scrobbling it to downstream service and adding to already seen scrobbles + +:::tip + +Keep in mind that modifying Scrobble/Play data earlier in the lifecycle will affect that data at all times later in the lifecycle. + +For example, to modify the track so it's the same anywhere it is processed in multi-scrobbler you only need to modify it in the **Source's** `preCompare` hook because all later processes will receive the data with the modified track. + +::: + +### Modification Parts + + +Each **hook** (`preCompare` etc...) is an object that specifies what part of the **Play** to modify: + +```json5 +{ + "title": [/* ... */], + "artists": [/* ... */], + "album": [/* ... */] +} +``` + +##### Expression + +and then a **list** what pattern/replacements (expressions) to use for the modification by using either simple strings or `search-replace` objects: + +```json5 +[ + "badTerm", // remove all instances of 'badTerm' + { + "search": "anotherBadTerm", // and also match all instances of 'anotherBadTerm' + "replace": "goodTerm" // replace with the string 'goodTerm' + } +] +``` + +Putting it all together: + +```json5 title="lastfm.json" +[ + { + "name": "myLastFm", + "enable": true, + "configureAs": "source", + "data": { + // ... + }, + "options": { + "playTransform": { + "preCompare": { + "title": [ + [ + "badTerm", + { + "search": "badTerm", + "replace": "goodTerm" + } + ] + ] + }, + } + } + } +] +``` + +#### Compare Hook + +The `compare` [hook](#hook) is slightly different than `preCompare` and `postCompare`. It consists of an object where you define which side(s) of the comparison should be modified. It also **does not modify downstream data!** Instead, the modifications are made only for use in the comparison. + +```json5 title="lastfm.json" +[ + { + "name": "myLastFm", + // ... + "options": { + "playTransform": { + "compare": { + "candidate": [/* ... */], // modify the "new" Play being compared + "existing": [/* ... */], // modify all "existing" Play/Scrobbles the new Play is being compared against + }, + } + } + } +] +``` + +#### Regular Expressions + +In addition to plain strings [expressions](#expression) that are matched and removed you can also use Regular Expressions. Write your regex like you normally would, but as a string, and it'll automatically be parsed: + +```json5 +[ + "/^\(\w+.com)/i", // matches any string that starts with '(YourMusic.com)' and removes it + { + "search": "/^\(\w+.com)/i", // matches any string that starts with '(YourMusic.com)' + "replace": "[MySite.com]" // replace with the string '[MySite.com]' + } +] +``` + +The `replace` property uses javascript's [`replace()` function and so can use any special string characters.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_the_replacement) + +## Examples + +### Remove phrase from Title everywhere + +Removes the phrase `(Album Version)` from the Title of a Play/Scrobble + +
+ + Example + +```json5 title="lastfm.json" +[ + { + "name": "myLastFm", + // ... + "options": { + "playTransform": { + "preCompare": { + "title": [ + [ + "(Album Version)" + ] + ] + }, + } + } + } +] +``` +
+ From 2bed7be654ccc1707e541c8daf36c2257900448d Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Thu, 25 Jul 2024 17:47:57 -0400 Subject: [PATCH 16/23] docs: Fix compare example --- docsite/docs/configuration/transforms.mdx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docsite/docs/configuration/transforms.mdx b/docsite/docs/configuration/transforms.mdx index 5699e9be..b8024c4d 100644 --- a/docsite/docs/configuration/transforms.mdx +++ b/docsite/docs/configuration/transforms.mdx @@ -159,8 +159,8 @@ The `compare` [hook](#hook) is slightly different than `preCompare` and `postCom "options": { "playTransform": { "compare": { - "candidate": [/* ... */], // modify the "new" Play being compared - "existing": [/* ... */], // modify all "existing" Play/Scrobbles the new Play is being compared against + "candidate": {/* ... */}, // modify the "new" Play being compared + "existing": {/* ... */}, // modify all "existing" Play/Scrobbles the new Play is being compared against }, } } @@ -203,9 +203,7 @@ Removes the phrase `(Album Version)` from the Title of a Play/Scrobble "playTransform": { "preCompare": { "title": [ - [ "(Album Version)" - ] ] }, } From fb3886b98173c02b605453ec2d6cc368977432c5 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 26 Jul 2024 09:19:11 -0400 Subject: [PATCH 17/23] docs: Modification improvements --- docsite/docs/configuration/transforms.mdx | 94 ++++++++++++++++++++--- 1 file changed, 85 insertions(+), 9 deletions(-) diff --git a/docsite/docs/configuration/transforms.mdx b/docsite/docs/configuration/transforms.mdx index b8024c4d..2c5df902 100644 --- a/docsite/docs/configuration/transforms.mdx +++ b/docsite/docs/configuration/transforms.mdx @@ -22,7 +22,7 @@ In any scenario where a repeating pattern can be found in the data it would be n First, let's recap the lifecycle of a scrobble in multi-scrobbler: -**Sources** are the beginning of the journey for a **Play** (song you've listened to long enough to scrobblable) +**Sources** are the beginning of the journey for a **Play** (song you've listened to long enough to be scrobblable) * A Source finds a new valid **Play** * The Source **compares** this new Play to all the other Plays it has already seen, if the Play is unique (title/artist/album/listened datetime) then... @@ -73,13 +73,13 @@ In more concrete terms this is the structure of hooks within a configuration (ca For **Sources**: * `preCompare` - modify Play data immediately when received -* `compare` - modify Play data when it is being compared to see if Play was already discovered +* `compare` - temporarily modify Play data when it is being compared to see if Play was already discovered * `postCompare` - modify Play data before sending to scrobble **Clients** For **Clients**: * `preCompare` - modify Play data immediately when received -* `compare` - modify Play data when it is being compared to see if it was already scrobbled +* `compare` - temporarily modify Play data when it is being compared to see if it was already scrobbled * `postCompare` - modify Play data before scrobbling it to downstream service and adding to already seen scrobbles :::tip @@ -93,7 +93,7 @@ For example, to modify the track so it's the same anywhere it is processed in mu ### Modification Parts -Each **hook** (`preCompare` etc...) is an object that specifies what part of the **Play** to modify: +Each [**hook**](#hook) (`preCompare` etc...) is an object that specifies what part of the **Play** to modify: ```json5 { @@ -147,6 +147,32 @@ Putting it all together: ] ``` +:::tip + +Modifications can also be applied to **all Sources** or **all Clients** when using the [AIO Config](./configuration.mdx?configType=aio#configuration-types) `config.json` by setting `playTransform` in `sourceDefaults` or `clientDefaults`: + +
+ + Example +```json5 title="config.json" +{ + "sourceDefaults": { // will apply playTransform to all sources + "playTransform": { + "preCompare": { + "title": [ + "(Album Version)" + ] + } + } + }, + "sources": [/* ... */], + "clients": [/* ... */] +} +``` +
+ +::: + #### Compare Hook The `compare` [hook](#hook) is slightly different than `preCompare` and `postCompare`. It consists of an object where you define which side(s) of the comparison should be modified. It also **does not modify downstream data!** Instead, the modifications are made only for use in the comparison. @@ -186,14 +212,35 @@ The `replace` property uses javascript's [`replace()` function and so can use an ## Examples -### Remove phrase from Title everywhere +### Remove phrase from Title in all new Plays + +Removes the phrase `(Album Version)` from the Title of a Play -Removes the phrase `(Album Version)` from the Title of a Play/Scrobble
Example +```json5 title="config.json" +{ + "sourceDefaults": { + "playTransform": { + "preCompare": { + "title": [ + "(Album Version)" + ] + } + } + } +} +``` +
+ +### Remove all parenthesized content from the end of a title + +
+ + Example ```json5 title="lastfm.json" [ { @@ -201,10 +248,17 @@ Removes the phrase `(Album Version)` from the Title of a Play/Scrobble // ... "options": { "playTransform": { - "preCompare": { + "compare": { + "candidate": { "title": [ - "(Album Version)" - ] + "/(\(.+\))\s*$/" + ] + }, + "existing": { + "title": [ + "/(\(.+\))\s*$/" + ] + }, }, } } @@ -213,3 +267,25 @@ Removes the phrase `(Album Version)` from the Title of a Play/Scrobble ```
+### Rename misspelled artist in all new Plays + +
+ + Example +```json5 title="config.json" +{ + "sourceDefaults": { + "playTransform": { + "preCompare": { + "artists": [ + { + "search": "Boz Skaggs", + "replace": "Boz Scaggs" + } + ] + } + } + } +} +``` +
From 1d276186f8bd4a4d68e2f13d0023a6910b7e2152 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 26 Jul 2024 09:30:58 -0400 Subject: [PATCH 18/23] feat: Implement removing field if field is an empty string after transforming --- docsite/docs/configuration/transforms.mdx | 29 +++++++++++ src/backend/common/AbstractComponent.ts | 30 +++-------- src/backend/tests/component/component.test.ts | 52 +++++++++++++++++++ 3 files changed, 89 insertions(+), 22 deletions(-) diff --git a/docsite/docs/configuration/transforms.mdx b/docsite/docs/configuration/transforms.mdx index 2c5df902..27167085 100644 --- a/docsite/docs/configuration/transforms.mdx +++ b/docsite/docs/configuration/transforms.mdx @@ -147,6 +147,12 @@ Putting it all together: ] ``` +:::note + +If the value of the field (title, an artist, album) is an empty string after transforming then the field is **removed.** + +::: + :::tip Modifications can also be applied to **all Sources** or **all Clients** when using the [AIO Config](./configuration.mdx?configType=aio#configuration-types) `config.json` by setting `playTransform` in `sourceDefaults` or `clientDefaults`: @@ -289,3 +295,26 @@ Removes the phrase `(Album Version)` from the Title of a Play } ``` + +### Remove "Various Artists" albums in all new Plays + +
+ + Example +```json5 title="config.json" +{ + "sourceDefaults": { + "playTransform": { + "preCompare": { + "album": [ + { + "search": "Various Artists", + "replace": "" + } + ] + } + } + } +} +``` +
diff --git a/src/backend/common/AbstractComponent.ts b/src/backend/common/AbstractComponent.ts index b50db2fb..61cd669a 100644 --- a/src/backend/common/AbstractComponent.ts +++ b/src/backend/common/AbstractComponent.ts @@ -274,16 +274,13 @@ export default abstract class AbstractComponent { const transformedPlayData: Partial = {}; - //const results: [string, string, string][] = []; let isTransformed = false; if (hook.title !== undefined && track !== undefined) { try { const t = searchAndReplace(track, hook.title); if (t !== track) { - //results.push(['title', track, t]); - transformedPlayData.track = t; - //transformedPlay.data.track = t; + transformedPlayData.track = t.trim() === '' ? undefined : t; isTransformed = true; } } catch (e) { @@ -301,7 +298,9 @@ export default abstract class AbstractComponent { anyArtistTransformed = true; isTransformed = true; } - transformedArtists.push(t); + if(t.trim() !== '') { + transformedArtists.push(t); + } } catch (e) { getLogger().warn(new Error(`Failed to transform artist: ${artist}`, {cause: e})); transformedArtists.push(artist); @@ -310,10 +309,6 @@ export default abstract class AbstractComponent { if(anyArtistTransformed) { transformedPlayData.artists = transformedArtists; } - //transformedPlay.data.artists = transformedArtists; - // if (anyTransformed) { - // //results.push(['artists', artists.join(' / '), transformedArtists.join(' / ')]); - // } } if (hook.artists !== undefined && albumArtists !== undefined && albumArtists.length > 0) { @@ -326,7 +321,9 @@ export default abstract class AbstractComponent { anyArtistTransformed = true; isTransformed = true; } - transformedArtists.push(t); + if(t.trim() !== '') { + transformedArtists.push(t); + } } catch (e) { getLogger().warn(new Error(`Failed to transform albumArtist: ${artist}`, {cause: e})); transformedArtists.push(artist); @@ -335,10 +332,6 @@ export default abstract class AbstractComponent { if(anyArtistTransformed) { transformedPlayData.albumArtists = transformedArtists; } - // transformedPlay.data.albumArtists = transformedArtists; - // if (anyTransformed) { - // results.push(['albumArtists', artists.join(' / '), transformedArtists.join(' / ')]); - // } } if (hook.album !== undefined && album !== undefined) { @@ -346,9 +339,7 @@ export default abstract class AbstractComponent { const t = searchAndReplace(album, hook.album); if (t !== album) { isTransformed = true; - transformedPlayData.album = t; - // results.push(['album', album, t]); - // transformedPlay.data.album = t; + transformedPlayData.album = t.trim() === '' ? undefined : t; } } catch (e) { getLogger().warn(new Error(`Failed to transform album: ${album}`, {cause: e})); @@ -376,11 +367,6 @@ Transformed : ${buildTrackString(transformedPlay, {include: ['artist', 'track', return transformedPlay; } -// if (results.length > 0 && log) { -// this.logger.debug({labels}, `Play transformed by ${hookType}: -// ${results.map(x => `${x[0]}: ${x[1]} => ${x[2]}`).join('\n')}`); -// } - return play; } catch (e) { getLogger().warn(new Error(`Unexpected error occurred, returning original play.`, {cause: e})); diff --git a/src/backend/tests/component/component.test.ts b/src/backend/tests/component/component.test.ts index 13f2be4f..8b98f7f5 100644 --- a/src/backend/tests/component/component.test.ts +++ b/src/backend/tests/component/component.test.ts @@ -165,6 +165,58 @@ describe('Play Transforms', function () { const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); expect(transformed.data.track).equal('My cool thing track'); }); + + it('Removes title when transform replaces with empty string', function () { + component.config = { + options: { + playTransform: { + preCompare: { + title: ['something'] + } + } + } + } + component.buildTransformRules(); + + const play = generatePlay({track: 'something'}); + const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + expect(transformed.data.track).is.undefined; + }); + + it('Removes album when transform replaces with empty string', function () { + component.config = { + options: { + playTransform: { + preCompare: { + album: ['something'] + } + } + } + } + component.buildTransformRules(); + + const play = generatePlay({album: 'something'}); + const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + expect(transformed.data.album).is.undefined; + }); + + it('Removes an artist when transform replaces with empty string', function () { + component.config = { + options: { + playTransform: { + preCompare: { + artists: ['something'] + } + } + } + } + component.buildTransformRules(); + + const play = generatePlay({artists: ['something', 'big']}); + const transformed = component.transformPlay(play, TRANSFORM_HOOK.preCompare); + expect(transformed.data.artists.length).is.eq(1) + expect(transformed.data.artists[0]).is.eq('big') + }); }); From 55ddd2c5a0d24fd499f85ccbbdc02df94c387448 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 26 Jul 2024 09:38:18 -0400 Subject: [PATCH 19/23] docs: Mention data modification in readme --- README.md | 3 ++- docsite/src/pages/index.mdx | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 761b1b50..bbab8f28 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * Graceful network and client failure handling (queued scrobbles that auto-retry) * Smart handling of credentials (persistent, authorization through app) * Easy configuration through ENVs or JSON -* Install using [Docker images for x86/ARM](https://foxxmd.github.io/multi-scrobbler/docs/installation#docker#docker), [flatpak](https://foxxmd.github.io/multi-scrobbler/docs/installation#docker#flatpak), or [locally with NodeJS](https://foxxmd.github.io/multi-scrobbler/docs/installation#docker#nodejs) +* Modify data before scrobbling with [regular expression or search patterns](https://foxxmd.github.io/multi-scrobbler/docs/transforms) +* Install using [Docker images for x86/ARM](https://foxxmd.github.io/multi-scrobbler/docs/installation#docker), [flatpak](https://foxxmd.github.io/multi-scrobbler/docs/installationr#flatpak), or [locally with NodeJS](https://foxxmd.github.io/multi-scrobbler/docs/installation#nodejs) [**Quick Start Guide**](https://foxxmd.github.io/multi-scrobbler/docs/quickstart) diff --git a/docsite/src/pages/index.mdx b/docsite/src/pages/index.mdx index c5ff95e8..75b7d17b 100644 --- a/docsite/src/pages/index.mdx +++ b/docsite/src/pages/index.mdx @@ -39,6 +39,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * Graceful network and client failure handling (queued scrobbles that auto-retry) * Smart handling of credentials (persistent, authorization through app) * Easy configuration through ENVs or JSON +* Modify data before scrobbling with [regular expression or search patterns](docs/transforms) * Install using [Docker images for x86/ARM](docs/installation#docker), [flatpak](docs/installation#flatpak), or [locally with NodeJS](docs/installation#nodejs) [**Quick Start Guide**](docs/quickstart) From 114a3dda8be82cbf89d489a884ecb2df243d4f62 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Fri, 26 Jul 2024 10:07:25 -0400 Subject: [PATCH 20/23] feat(ui): Add timestamp to recent pages #167 --- src/client/recent/RecentPage.tsx | 54 +++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/src/client/recent/RecentPage.tsx b/src/client/recent/RecentPage.tsx index c59338f8..c09b9d45 100644 --- a/src/client/recent/RecentPage.tsx +++ b/src/client/recent/RecentPage.tsx @@ -1,22 +1,33 @@ -import React, {Fragment} from 'react'; +import React, { Fragment, useMemo } from 'react'; import PlayDisplay from "../components/PlayDisplay"; -import {recentIncludes} from "../../core/Atomic"; -import {useSearchParams} from "react-router-dom"; -import {useGetRecentQuery} from "./recentDucks"; +import { recentIncludes } from "../../core/Atomic"; +import { useSearchParams } from "react-router-dom"; +import { useGetRecentQuery } from "./recentDucks"; import Tooltip from "../components/Tooltip"; -import {faQuestionCircle} from "@fortawesome/free-solid-svg-icons"; -import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; -import {data} from "autoprefixer"; +import { faQuestionCircle } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; const displayOpts = { include: recentIncludes, includeWeb: true } -const apiTip = +const apiTipContent =
Data that is directly returned by the Source API.
-
If you do not see your recent plays in this data it is likely the Source's data is lagging behind your actual activity.
- +
If you do not see your recent plays in this data it is likely the Source's data is lagging behind your + actual activity. +
+
+; + +const tsTip =
+
+ (C) - Scrobble timestamped when listen was completed +
+
+ (S) - Scrobble timestamped when listen was started +
+
; const recent = () => { let [searchParams, setSearchParams] = useSearchParams(); @@ -25,17 +36,32 @@ const recent = () => { error, isLoading, isSuccess - } = useGetRecentQuery({name: searchParams.get('name'), type: searchParams.get('type'), upstream: searchParams.get('upstream')}); + } = useGetRecentQuery({ + name: searchParams.get('name'), + type: searchParams.get('type'), + upstream: searchParams.get('upstream') + }); const isUpstream = searchParams.get('upstream') === '1'; + const tipContents = useMemo(() => { + return + {isUpstream ? apiTipContent : null} + {tsTip} + + }, [isUpstream]); + return (
-

Recently Played{isUpstream ? ' from Source API' : null}{isUpstream ? : null} +

Recently Played{isUpstream ? ' from Source API' : null}

From 831cefd9a72852cc606923ec5c822368999d0ea4 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 19 Aug 2024 11:25:11 -0400 Subject: [PATCH 21/23] docs(deezer): Deprecate due to discontinued API support Deprecate Source with warning message and update docs. #175 --- README.md | 2 +- docsite/docs/FAQ.md | 4 ++++ docsite/docs/configuration/configuration.mdx | 18 ++++++++++++++++-- docsite/src/pages/index.mdx | 2 +- ...o.github.foxxmd.multiscrobbler.metainfo.xml | 1 - src/backend/sources/DeezerSource.ts | 2 ++ 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bbab8f28..a7fe5eb3 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * [Youtube Music](https://foxxmd.github.io/multi-scrobbler/docs/configuration#youtube-music) * [Last.fm](https://foxxmd.github.io/multi-scrobbler/docs/configuration#lastfm-source) * [ListenBrainz](https://foxxmd.github.io/multi-scrobbler/docs/configuration#listenbrainz-source) - * [Deezer](https://foxxmd.github.io/multi-scrobbler/docs/configuration#deezer) + * [~~Deezer~~](https://foxxmd.github.io/multi-scrobbler/docs/configuration#deezer) * [MPRIS (Linux Desktop)](https://foxxmd.github.io/multi-scrobbler/docs/configuration#mpris) * [Mopidy](https://foxxmd.github.io/multi-scrobbler/docs/configuration#mopidy) * [JRiver](https://foxxmd.github.io/multi-scrobbler/docs/configuration#jriver) diff --git a/docsite/docs/FAQ.md b/docsite/docs/FAQ.md index 9114799c..897f51ac 100644 --- a/docsite/docs/FAQ.md +++ b/docsite/docs/FAQ.md @@ -124,6 +124,10 @@ If multi-scrobbler is not running on the same machine your browser is on then th EX `http://localhost:9078/lastfm/callback` -> `http://192.168.0.220:9078/lastfm/callback` +### Deezer is not working + +Deezer has discontinued support for their API and the Deezer Source is now [**deprecated.**](configuration/configuration.mdx#deezer) See [this issue for more discussion.](https://github.com/FoxxMD/multi-scrobbler/issues/175#issuecomment-2296776625) + ## Configuration Issues ### Config could not be parsed diff --git a/docsite/docs/configuration/configuration.mdx b/docsite/docs/configuration/configuration.mdx index 7b9d3026..d0f1f6ed 100644 --- a/docsite/docs/configuration/configuration.mdx +++ b/docsite/docs/configuration/configuration.mdx @@ -147,7 +147,7 @@ These options affect multi-scrobbler's behavior and are not specific to any sour #### Base URL -Defines the URL that is used to generate default redirect URLs for authentication on [spotify](#spotify), [lastfm](#lastfm), and [deezer](#deezer) -- as well as some logging hints. +Defines the URL that is used to generate default redirect URLs for authentication on [spotify](#spotify) and [lastfm](#lastfm) -- as well as some logging hints. * Default => `http://localhost:9078` * Set with [ENV](./configuration?configType=env#configuration-types) `BASE_URL` or `baseUrl` [all-in-one configuration](./configuration?configType=aio#configuration-types) @@ -492,7 +492,21 @@ On your [profile page](https://listenbrainz.org/profile/) find your **User Token -### [Deezer](https://deezer.com/) +### [~~Deezer~~](https://deezer.com/) + +:::warning + +**This Source is DEPRECATED because Deezer has dropped official API support.** This Source will **not** be removed but no further support or fixes will be given. + +Users cannot create new applications on Deezer Developers and there is no guarantee existing applications will continue to work. + +As a workaround consider integrating Deezer with last.fm and then using [last.fm as a Source](#lastfm-source). + +Users with existing Deezer applications in use with multi-scrobbler should consider this change as well to avoid future breaking issues with the unsupported API. + +[See this issue for more discussion.](https://github.com/FoxxMD/multi-scrobbler/issues/175#issuecomment-2296776625) + +::: Create a new application at [Deezer Developers](https://developers.deezer.com/myapps) diff --git a/docsite/src/pages/index.mdx b/docsite/src/pages/index.mdx index 75b7d17b..21d04d43 100644 --- a/docsite/src/pages/index.mdx +++ b/docsite/src/pages/index.mdx @@ -21,7 +21,7 @@ A javascript app to scrobble music you listened to, to [Maloja](https://github.c * [Youtube Music](docs/configuration#youtube-music) * [Last.fm](docs/configuration#lastfm-source) * [ListenBrainz](docs/configuration#listenbrainz-source) - * [Deezer](docs/configuration#deezer) + * [~~Deezer~~](docs/configuration#deezer) * [MPRIS (Linux Desktop)](docs/configuration#mpris) * [Mopidy](docs/configuration#mopidy) * [JRiver](docs/configuration#jriver) diff --git a/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml b/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml index 42531010..65a09307 100644 --- a/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml +++ b/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml @@ -26,7 +26,6 @@
  • Jellyfin
  • WebScrobbler
  • Mopidy
    • -
  • Deezer
  • JRiver
  • Kodi
  • Webscrobbler
    • diff --git a/src/backend/sources/DeezerSource.ts b/src/backend/sources/DeezerSource.ts index dbe022d7..ee4db3a3 100644 --- a/src/backend/sources/DeezerSource.ts +++ b/src/backend/sources/DeezerSource.ts @@ -90,6 +90,8 @@ export default class DeezerSource extends AbstractSource { } protected async doBuildInitData(): Promise { + this.logger.warn('This Source is DEPRECATED! Deezer has dropped support official API support. New apps cannot be created and existing apps are not guaranteed to continue working. See the documentation or this issue for more information: https://github.com/FoxxMD/multi-scrobbler/issues/175#issuecomment-2296776625'); + try { const credFile = await readJson(this.workingCredsPath, {throwOnNotFound: false}); if(credFile !== undefined) { From 0fed91302c32f92df390c482e5aff507f7eca865 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Mon, 19 Aug 2024 11:55:47 -0400 Subject: [PATCH 22/23] chore(ci): Update untagged package schedule to run once a week Doesn't need to run every day. --- .github/workflows/packagesDeleteUntagged.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/packagesDeleteUntagged.yml b/.github/workflows/packagesDeleteUntagged.yml index 9f01f1a3..0eefdcde 100644 --- a/.github/workflows/packagesDeleteUntagged.yml +++ b/.github/workflows/packagesDeleteUntagged.yml @@ -1,7 +1,7 @@ name: Delete Untagged Packages on: schedule: - - cron: '30 1 * * *' + - cron: '30 1 * * 0' workflow_run: workflows: ["Publish Docker image to Dockerhub"] types: From 4f26b63ab1a5076aaf28b757f0d1b5e8b261f001 Mon Sep 17 00:00:00 2001 From: FoxxMD Date: Tue, 20 Aug 2024 09:17:28 -0400 Subject: [PATCH 23/23] chore: Bump version for release --- flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml | 1 + package-lock.json | 12 ++++++------ package.json | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml b/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml index 65a09307..0126481b 100644 --- a/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml +++ b/flatpak/io.github.foxxmd.multiscrobbler.metainfo.xml @@ -49,6 +49,7 @@ + diff --git a/package-lock.json b/package-lock.json index 303bfd6a..3f3b88f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "multi-scrobbler", - "version": "0.8.1", + "version": "0.8.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "multi-scrobbler", - "version": "0.8.1", + "version": "0.8.2", "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -79,7 +79,7 @@ "devDependencies": { "@dbus-types/notifications": "^0.0.5", "@eslint/js": "^8.56.0", - "@faker-js/faker": "^9.0.0-rc.0", + "@faker-js/faker": "^9.0.0-rc.1", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^13.4.0", @@ -1121,9 +1121,9 @@ } }, "node_modules/@faker-js/faker": { - "version": "9.0.0-rc.0", - "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-9.0.0-rc.0.tgz", - "integrity": "sha512-8ouDqdlHqREHizwGk1QiXuL73vtBfRFh0Uh71aJyotCg6QMF9IgyvIsxf/IunoEr3gqpUwAlrhLMwLIvRK1mzg==", + "version": "9.0.0-rc.1", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-9.0.0-rc.1.tgz", + "integrity": "sha512-d9uL+O7Bud4W0axuHYxjCsfwazwo6iPRbgDkIPgWhtxpnziHqbFafOuKubnQ++4V31dI9NYYkajzGNJADHas4A==", "dev": true, "funding": [ { diff --git a/package.json b/package.json index 425beefe..0c321b65 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "multi-scrobbler", - "version": "0.8.1", + "version": "0.8.2", "type": "module", "description": "scrobble plays from multiple sources to multiple clients", "scripts": { @@ -114,7 +114,7 @@ "devDependencies": { "@dbus-types/notifications": "^0.0.5", "@eslint/js": "^8.56.0", - "@faker-js/faker": "^9.0.0-rc.0", + "@faker-js/faker": "^9.0.0-rc.1", "@istanbuljs/nyc-config-typescript": "^1.0.2", "@testing-library/jest-dom": "^5.17.0", "@testing-library/react": "^13.4.0",