From 1499d7486c3bd238dd264f1e8e0e301182fd8091 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Sat, 5 Oct 2024 21:25:50 -0700 Subject: [PATCH] [FEATURE] Override variable support in ytdl_options (#1087) Adds the ability to use override variables in the [ytdl_options](https://ytdl-sub.readthedocs.io/en/latest/config_reference/plugins.html#ytdl-options) section of the config. --- docs/source/prebuilt_presets/helpers.rst | 33 ++++++--- src/ytdl_sub/config/overrides.py | 72 +++++++++++++------ src/ytdl_sub/config/preset_options.py | 6 +- .../helpers/download_deletion_options.yaml | 6 +- .../subscription_ytdl_options.py | 6 +- tests/e2e/bandcamp/test_bandcamp.py | 6 +- 6 files changed, 93 insertions(+), 36 deletions(-) diff --git a/docs/source/prebuilt_presets/helpers.rst b/docs/source/prebuilt_presets/helpers.rst index fa6d25f9f..749c4c025 100644 --- a/docs/source/prebuilt_presets/helpers.rst +++ b/docs/source/prebuilt_presets/helpers.rst @@ -69,19 +69,34 @@ Supports the following override variables: - "To Catch a Smuggler" -Chunk Initial Download ----------------------- +Chunk Downloads +--------------- -If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from newest to oldest before starting any downloads. It is a long process and not ideal. A better method is to chunk the process by using the following preset: +If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from newest to oldest before +starting any downloads. It is a long process and not ideal. A better method is to chunk the process by using the +following preset: -``chunk_initial_download`` +``Chunk Downloads`` -It will download videos starting from the oldest one, and only download 20 at a time. You can -change this number by setting: +It will download videos starting from the oldest one, and only download 20 at a time by default. You can +change this number by setting the override variable ``chunk_max_downloads``. .. code-block:: yaml - ytdl_options: - max_downloads: 30 # Desired number to download per invocation + __preset__: + overrides: + chunk_max_downloads: 20 + + Plex TV Show by Date: + + # Chunk these ones + = Documentaries | Chunk Downloads: + "NOVA PBS": "https://www.youtube.com/@novapbs" + "National Geographic": "https://www.youtube.com/@NatGeo" + + # But not these ones + = Documentaries: + "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U" -Once the entire channel is downloaded, remove this preset. Then it will pull metadata from newest to oldest again, and stop pulling additional metadata once it reaches a video that has already been downloaded. \ No newline at end of file +Once the entire channel is downloaded, remove the usage of this preset. It will then pull metadata from newest to +oldest again, and stop once it reaches a video that has already been downloaded. \ No newline at end of file diff --git a/src/ytdl_sub/config/overrides.py b/src/ytdl_sub/config/overrides.py index 2fd930e01..9b7a43722 100644 --- a/src/ytdl_sub/config/overrides.py +++ b/src/ytdl_sub/config/overrides.py @@ -11,12 +11,14 @@ from ytdl_sub.entries.variables.override_variables import OverrideHelpers from ytdl_sub.script.parser import parse from ytdl_sub.script.script import Script +from ytdl_sub.script.types.resolvable import Resolvable from ytdl_sub.script.utils.exceptions import ScriptVariableNotResolved from ytdl_sub.utils.exceptions import InvalidVariableNameException from ytdl_sub.utils.exceptions import StringFormattingException from ytdl_sub.utils.exceptions import ValidationException from ytdl_sub.utils.script import ScriptUtils from ytdl_sub.utils.scriptable import Scriptable +from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator @@ -144,11 +146,36 @@ def initialize_script(self, unresolved_variables: Set[str]) -> "Overrides": self.update_script() return self + def _apply_to_resolvable( + self, + formatter: StringFormatterValidator, + entry: Optional[Entry], + function_overrides: Optional[Dict[str, str]], + ) -> Resolvable: + script: Script = self.script + unresolvable: Set[str] = self.unresolvable + if entry: + script = entry.script + unresolvable = entry.unresolvable + + try: + return script.resolve_once( + dict({"tmp_var": formatter.format_string}, **(function_overrides or {})), + unresolvable=unresolvable, + )["tmp_var"] + except ScriptVariableNotResolved as exc: + raise StringFormattingException( + "Tried to resolve the following script, but could not due to unresolved " + f"variables:\n {formatter.format_string}\n" + "This is most likely due to circular dependencies in variables. " + "If you think otherwise, please file a bug on GitHub and post your config. Thanks!" + ) from exc + def apply_formatter( self, formatter: StringFormatterValidator, entry: Optional[Entry] = None, - function_overrides: Dict[str, str] = None, + function_overrides: Optional[Dict[str, str]] = None, ) -> str: """ Parameters @@ -169,25 +196,28 @@ def apply_formatter( StringFormattingException If the formatter that is trying to be resolved cannot """ - script: Script = self.script - unresolvable: Set[str] = self.unresolvable - if entry: - script = entry.script - unresolvable = entry.unresolvable - - try: - return formatter.post_process( - str( - script.resolve_once( - dict({"tmp_var": formatter.format_string}, **(function_overrides or {})), - unresolvable=unresolvable, - )["tmp_var"] + return formatter.post_process( + str( + self._apply_to_resolvable( + formatter=formatter, entry=entry, function_overrides=function_overrides ) ) - except ScriptVariableNotResolved as exc: - raise StringFormattingException( - "Tried to resolve the following script, but could not due to unresolved " - f"variables:\n {formatter.format_string}\n" - "This is most likely due to circular dependencies in variables. " - "If you think otherwise, please file a bug on GitHub and post your config. Thanks!" - ) from exc + ) + + def apply_overrides_formatter_to_native( + self, + formatter: OverridesStringFormatterValidator, + ) -> Any: + """ + Parameters + ---------- + formatter + Overrides formatter to apply + + Returns + ------- + The native python form of the resolved variable + """ + return self._apply_to_resolvable( + formatter=formatter, entry=None, function_overrides=None + ).native diff --git a/src/ytdl_sub/config/preset_options.py b/src/ytdl_sub/config/preset_options.py index a6737a645..2d907b29d 100644 --- a/src/ytdl_sub/config/preset_options.py +++ b/src/ytdl_sub/config/preset_options.py @@ -9,11 +9,13 @@ from ytdl_sub.validators.string_formatter_validators import OverridesIntegerFormatterValidator from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator +from ytdl_sub.validators.string_formatter_validators import ( + UnstructuredOverridesDictFormatterValidator, +) from ytdl_sub.validators.validators import BoolValidator -from ytdl_sub.validators.validators import LiteralDictValidator -class YTDLOptions(LiteralDictValidator): +class YTDLOptions(UnstructuredOverridesDictFormatterValidator): """ Allows you to add any ytdl argument to ytdl-sub's downloader. The argument names can differ slightly from the command-line argument names. See diff --git a/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml b/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml index b62f73cdf..676862a36 100644 --- a/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml +++ b/src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml @@ -37,10 +37,12 @@ presets: chunk_initial_download: # legacy preset name ytdl_options: - max_downloads: 20 + max_downloads: "{chunk_max_downloads}" playlistreverse: True break_on_existing: False + overrides: + chunk_max_downloads: 20 - "Download in Chunks": + "Chunk Downloads": preset: - chunk_initial_download \ No newline at end of file diff --git a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py index 1cc6f3174..d1f288eea 100644 --- a/src/ytdl_sub/subscriptions/subscription_ytdl_options.py +++ b/src/ytdl_sub/subscriptions/subscription_ytdl_options.py @@ -110,7 +110,11 @@ def _plugin_ytdl_options(self, plugin: Type[PluginT]) -> Dict: @property def _user_ytdl_options(self) -> Dict: - return self._preset.ytdl_options.dict + native_ytdl_options = { + key: self._overrides.apply_overrides_formatter_to_native(val) + for key, val in self._preset.ytdl_options.dict.items() + } + return native_ytdl_options @property def _plugin_match_filters(self) -> Dict: diff --git a/tests/e2e/bandcamp/test_bandcamp.py b/tests/e2e/bandcamp/test_bandcamp.py index ee90e065e..c4d410cd9 100644 --- a/tests/e2e/bandcamp/test_bandcamp.py +++ b/tests/e2e/bandcamp/test_bandcamp.py @@ -12,7 +12,10 @@ def subscription_dict(output_directory): return { "preset": "Bandcamp", "ytdl_options": { - "max_downloads": 15, + # Test that ytdl-options can handle overrides + # TODO: Move this to a local test + "max_downloads": "{max_downloads}", + "extractor_args": {"youtube": {"lang": ["en"]}}, }, "audio_extract": {"codec": "mp3", "quality": 320}, "date_range": {"after": "20210110"}, @@ -20,6 +23,7 @@ def subscription_dict(output_directory): "subscription_value": "https://sithuayemusic.bandcamp.com/", "subscription_indent_1": "Progressive Metal", "music_directory": output_directory, + "max_downloads": 15, }, }