From 7e4b17a4153083c25c863a4875cb1e850d3b498f Mon Sep 17 00:00:00 2001 From: Hydrus Network Developer Date: Wed, 25 Sep 2024 16:06:44 -0500 Subject: [PATCH] Version 591 --- docs/changelog.md | 84 +- docs/developer_api.md | 116 +- docs/old_changelog.html | 31 + hydrus/client/ClientAPI.py | 78 +- hydrus/client/ClientConstants.py | 4 +- hydrus/client/ClientController.py | 2 +- hydrus/client/ClientFiles.py | 5 + hydrus/client/ClientMigration.py | 21 +- hydrus/client/ClientOptions.py | 2 +- hydrus/client/db/ClientDB.py | 89 + hydrus/client/duplicates/ClientDuplicates.py | 16 +- hydrus/client/gui/ClientGUI.py | 2 +- hydrus/client/gui/ClientGUIAPI.py | 42 +- hydrus/client/gui/canvas/ClientGUICanvas.py | 2 - .../client/gui/media/ClientGUIMediaMenus.py | 70 +- .../gui/metadata/ClientGUIMigrateTags.py | 97 +- hydrus/client/gui/pages/ClientGUIPages.py | 40 +- hydrus/client/gui/pages/ClientGUIResults.py | 576 +- .../gui/panels/ClientGUIManageOptionsPanel.py | 6 +- .../client/importing/ClientImportFileSeeds.py | 2 +- hydrus/client/networking/ClientLocalServer.py | 167 - .../networking/ClientLocalServerResources.py | 5016 ----------------- .../networking/api/ClientLocalServer.py | 182 + .../networking/api/ClientLocalServerCore.py | 1053 ++++ .../api/ClientLocalServerResources.py | 200 + .../api/ClientLocalServerResourcesAccess.py | 271 + .../api/ClientLocalServerResourcesAddFiles.py | 360 ++ .../api/ClientLocalServerResourcesAddNotes.py | 127 + .../api/ClientLocalServerResourcesAddTags.py | 399 ++ .../api/ClientLocalServerResourcesAddURLs.py | 318 ++ .../ClientLocalServerResourcesEditRatings.py | 112 + .../ClientLocalServerResourcesEditTimes.py | 157 + .../api/ClientLocalServerResourcesGetFiles.py | 907 +++ ...ClientLocalServerResourcesManageCookies.py | 390 ++ ...lientLocalServerResourcesManageDatabase.py | 166 + ...lServerResourcesManageFileRelationships.py | 299 + .../ClientLocalServerResourcesManagePages.py | 167 + .../ClientLocalServerResourcesManagePopups.py | 361 ++ ...lientLocalServerResourcesManageServices.py | 107 + hydrus/client/networking/api/__init__.py | 0 hydrus/core/HydrusConstants.py | 8 +- hydrus/core/networking/HydrusServerRequest.py | 2 +- hydrus/test/TestClientAPI.py | 121 +- hydrus/test/TestClientMigration.py | 265 +- hydrus/test/TestController.py | 5 +- hydrus/test/TestHydrusServer.py | 1 - 46 files changed, 6773 insertions(+), 5673 deletions(-) delete mode 100644 hydrus/client/networking/ClientLocalServer.py delete mode 100644 hydrus/client/networking/ClientLocalServerResources.py create mode 100644 hydrus/client/networking/api/ClientLocalServer.py create mode 100644 hydrus/client/networking/api/ClientLocalServerCore.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResources.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesAccess.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesAddFiles.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesAddNotes.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesAddTags.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesAddURLs.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesEditRatings.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesEditTimes.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesGetFiles.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesManageCookies.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesManageDatabase.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesManageFileRelationships.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesManagePages.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesManagePopups.py create mode 100644 hydrus/client/networking/api/ClientLocalServerResourcesManageServices.py create mode 100644 hydrus/client/networking/api/__init__.py diff --git a/docs/changelog.md b/docs/changelog.md index 358fdc082..a89f834e9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,42 @@ title: Changelog !!! note This is the new changelog, only the most recent builds. For all versions, see the [old changelog](old_changelog.html). +## [Version 591](https://github.com/hydrusnetwork/hydrus/releases/tag/v591) + +### misc + +* fixed a stupid oversight with last week's "move page focus left/right after closing tab" thing where it was firing even when the page closed was not the current tab!! it now correctly only moves your focus if you close the _current_ tab, not if you just middle click some other one +* fixed the _share->export files_ menu command not showing if you right-clicked on just one file +* cleaned some of the broader thumbnail menu code, separating the 'stuff to show if we have a focus' and 'stuff to show if we have a selection'; the various 'manage' commands now generally show even if there is no current 'focus' in the preview (which happens if you select with ctrl+click or ctrl+a and then right-click in whitespace) +* the 'migrate tags' dialog now allows you to filter the sibling or parent pairs by whether the child/worse or parent/ideal tag has actual mapping counts on an arbitrary tag service. some new unit tests ensure this capability +* fixed an error in the duplicate metadata merge system where if files were exchanging known URLs, and one of those URLs was not actually an URL (e.g. it was garbage data, or human-entered 'location' info), a secondary system that tried to merge correlated domain-based timestamps was throwing an exception +* to reduce comma-confusion, the template for 'show num files and import status' on page names is now "name - (num_files - import_status)" +* the option that governs whether page names have the file count after them (under _options->gui pages_) has a new choice--'show for all pages, but only if greater than zero'--which is now the default for new users + +### some boring code cleanup + +* broke up the over-coupled 'migrate tags' unit tests into separate content types and the new count-filtering stuff +* cleaned up the 'share' menu construction code--it was messy after some recent rewrites +* added some better error handling around some of the file/thumbnail path fetching/regen routines + +### client api + +* the client api gets a new permissions state this week: the permissions structure you edit for an access key can now be (and, as a convenient default, starts as) a simple 'permits everything' state. if the permissions are set to 'permit everything', then this overrules all the specific rules and tag search filter gubbins. nice and simple, and a permissions set this way will automatically inherit new permissions in the future. any api access keys that have all the permissions up to 'edit ratings' will be auto-updated to 'permits everything' and you will get an update saying this happened--check your permissions in _review services_ if you need finer control +* added a new permission, `13`, for 'see local paths' +* added `/get_files/file_path`, which fetches the local path of a file. it needs the new permission +* added `/get_files/thumbnail_path`, which fetches the local path of a thumbnail and optionally the filetype of the actual thumb (jpeg or png). it needs the new permission +* the `/request_new_permissions` command now accepts a `permits_everything` bool as a selective alternate to the `basic_permissions` list +* the `/verify_access_key` command now responds with the name of the access key and the new `permits_everything` value +* the API help is updated for the above +* new unit tests test all the above +* the Client API version is now 71 + +### client api refactoring + +* the main `ClientLocalServerResources` file has been getting too huge (5,000 lines), so I've moved it and `ClientLocalServer` to their own `api` module and broken the Resources file up into core functions, the superclass, and the main verbs +* fixed permissions check for `/manage_popups/update_popup`, which was checking for pages permission rather than popup permission +* did a general linting pass of these easier-to-handle files; cleaned up some silly stuff + ## [Version 590](https://github.com/hydrusnetwork/hydrus/releases/tag/v590) ### misc @@ -338,51 +374,3 @@ title: Changelog * when you enter a wildcard into a Read tag autocomplete, it no longer always delivers the 'always autocompleting' version. so, if you enter `sa*s`, it will suggest `sa*s (wildcard search)` and perhaps `sa*s (any namespace)`, but it will no longer suggest the `sa*s*` variants until you, obviously, actually type that trailing asterisk yourself. I intermittently had no idea what the hell I was doing when I originally developed this stuff * the 'unnamespaced input gives `(any namespace)` wildcard results' tag display option is now correctly negatively enforced when entering unnamespaced wildcards. previously it was always adding them, and sometimes inserting them at the top of the list. the `(any namespace)` variant is now always below the unnamespaced when both are present * fixed up a bunch of jank unit tests that were testing this badly - -## [Version 581](https://github.com/hydrusnetwork/hydrus/releases/tag/v581) - -### misc - -* thanks to a user, we have a much improved shimmie parser, for both file and gallery urls, that fetches md5 better, improves gallery navigation, stops grabbing bad urls and related tags by accident, and can handle namespaces for those shimmies that use them. for our purposes, this improves r34h and r34@paheal downloaders by default -* thanks to a user, we have a new 'Dark Blue 1.1' styesheet with some improvements. the recommendation is: check the different scrollbar styling to see if you prefer the old version -* timedelta widgets now enforce their minimum time on focus-out rather than value change. if it wants at least 20 minutes, you can now type in '5...' in the minutes column without it going nuts. let me know if you discover a way to out-fox the focus-out detection! -* added a checkbox to file import options to govern whether 'import destinations' and 'archive all imports' apply to 'already in db' files. this turns on/off the logic that I made more reliable last week. default is that they do -* added 'do sleep check' to _options->system_ to try some things out on systems that often false-positive this check -* the 'review current network jobs' multi-column list has a new right-click menu to show a bit more debug info about each job: each of its network contexts, how the bandwidth is on each context, if the domain is ok, if it is waiting on a connection error, if it is waiting on serverside bandwidth, if it obey bandwidth, and if its tokens are ok. if you have been working with me on gallery jobs that just sit on 'starting soon', please check it out and let me know what you see. also, 'review current network jobs' is duplicated to the help->debug menu. I forgot where it was, so let's have it in both places -* on the filename-import tagging panel, the filename and directory checkbox-and-text-edit widgets no longer emit a (sometimes laggy) update signal when typing when the checkbox is unchecked - -### janitor stuff - -* if you are a repository janitor, right-clicking on any tag shows a new 'admin' menu -* if you have 'change options' permission, you will see 'block x'/'re-allow x' to let you quickly see if tags are blocked and then edit the repository tag filter respectively -* if you have 'mappings petition resolution' permission, you can 'purge' the selected tags, which will deleted them from the service entirely. this launches a review window that previews the job and allows adding of more tags using the standard autocomplete interface. when 'fired off', it launches a tag migration job to queue up the full petition/delete upload -* this new 'purge' window is also available from the normal 'administrate services' menu in the main gui -* also under the 'administrate services' is a new 'purge tag filter' command, which applies the existing repository tag filter to its own mappings store, retroactively syncing you to it - -### tag filters and migration - -* I wrote a database routine that quickly converts a hydrus tag filter into the list of tags within a file and tag search context. this tech will have a variety of uses in the genre of 'hey please delete/fetch/check all these tags' -* to start with, it is now plugged into the tag migration system, so when you set up, say, an 'all known files' tag migration that only looks for a namespace or a bunch of single tags, the 'setup' phase is now massively, massively faster (previously, with something like the PTR, this would be scanning through tens of millions of files for minutes; now it just targets the 50k or whatever using existing tag search tech usually within less than a second) -* cleaned (KISSed) and reworked the tag filter logic a bit--it can now, underlyingly, handle 'no namespaced tags, except for creator:anything, but still allowing creator:blah' -* optimised how tag filters do 'apply unnamespaced rules to namespaced tags' (which happens in some blacklists that want to be expansive) -* improved how the tag filter describes itself in many cases. it should make more grammatical sense and repeat itself less now (e.g. no more 'all tags and namespaced tags and unnamespaced tags' rubbish) -* improved how some tag filter rules are handled across the program, including fixing some edge-case false-positive namespace-rule detection -* deleted some ancient and no longer used tag filtering code - -### boring multi-column list stuff - -* did more 'select, sort, and scroll' code cleanup in my multi-column lists, specifically: manage import folders; manage export folders; the string-to-string dict list; edit ngug; edit downloader display (both gugs and url classes, and with a one-shot show/hide choice on a multi-selection rather than asking for each in turn); the special 'duplicate' command of edit shortcut set; and the string converter conversions list (including better select logic on move up/down) -* in keeping with the new general policy of 'when you edit a multi-column list, you just edit one row', the various 'edit' buttons under these lists across the program are now generally only enabled when you have one row selected -* the new 'select, sort, and scroll to new item when a human adds it' tech now _deselects_ the previous selection. let me know if this screws up anywhere (maybe in a hacky multi-add somewhere it'll only select the last added?) -* the aggravating 'clear the focus of the list on most changes bro' jank seems to be fixed--it was a dumb legacy thing -* whenever the multi-column list does its new 'scroll-to' action, it now takes focus to better highlight where we are (rather than stay, for instance, leaving focus on the 'add' button you just clicked) - -### other boring stuff - -* worked a little more on a routine that collapses an arbitrary list of strings to a human-presentable summary and replaced the hardcoded hacky version that presents the 'paste queries' result in the 'edit subscription' panel with it -* wrote a similar new routine to collapse an arbitrary list of strings to a single-line summary, appropriate for menu labels and such -* fixed a layout issue in the 'manage downloader display' dialog that caused the 'edit' button on the 'media viewer urls' side to not show, lmaooooooo -* ephemeral 'watcher' and 'gallery' network contexts now describe themselves with a nicer string -* decoupled how some service admin stuff works behind the scenes to make it easier to launch this stuff from different UI widgets -* refactored `ToHumanInt` and the `ToPrettyOrdinalString` guys to a new `HydrusNumbers.py` file -* fixed some bad Client API documentation for the params in `/get_files/search_files` diff --git a/docs/developer_api.md b/docs/developer_api.md index cd1a80dad..9cb80f655 100644 --- a/docs/developer_api.md +++ b/docs/developer_api.md @@ -86,7 +86,7 @@ If the client does not support CBOR, you'll get 406. ## Access and permissions -The client gives access to its API through different 'access keys', which are the typical 64-character hex used in many other places across hydrus. Each guarantees different permissions such as handling files or tags. Most of the time, a user will provide full access, but do not assume this. If the access header or parameter is not provided, you will get 401, and all insufficient permission problems will return 403 with appropriate error text. +The client gives access to its API through different 'access keys', which are the typical random 64-character hex used in many other places across hydrus. Each guarantees different permissions such as handling files or tags. Most of the time, a user will provide full access, but do not assume this. If a access key header or parameter is not provided, you will get 401, and all insufficient permission problems will return 403 with appropriate error text. Access is required for every request. You can provide this as an http header, like so: @@ -94,7 +94,9 @@ Access is required for every request. You can provide this as an http header, li Hydrus-Client-API-Access-Key : 0150d9c4f6a6d2082534a997f4588dcf0c56dffe1d03ffbf98472236112236ae ``` -Or you can include it in the normal parameters of any request (except _POST /add\_files/add\_file_, which uses the entire POST body for the file's bytes). For GET, this means including it into the URL parameters: +Or you can include it in the normal parameters of any request (except _POST /add\_files/add\_file_, which uses the entire POST body for the file's bytes). + +For GET, this means including it into the URL parameters: ``` /get_files/thumbnail?file_id=452158&Hydrus-Client-API-Access-Key=0150d9c4f6a6d2082534a997f4588dcf0c56dffe1d03ffbf98472236112236ae @@ -389,7 +391,8 @@ Required Headers: n/a Arguments: : * `name`: (descriptive name of your access) - * `basic_permissions`: A JSON-encoded list of numerical permission identifiers you want to request. + * `permits_everything`: (selective, bool, whether to permit all tasks now and in future) + * `basic_permissions`: Selective. A JSON-encoded list of numerical permission identifiers you want to request. The permissions are currently: @@ -406,11 +409,17 @@ Arguments: * 10 - Manage Popups * 11 - Edit File Times * 12 - Commit Pending - - ``` title="Example request" - /request_new_permissions?name=my%20import%20script&basic_permissions=[0,1] - ``` - + * 13 - See Local Paths + + +``` title="Example request" +/request_new_permissions?name=migrator&permit_everything=true +``` + +``` title="Example request (for permissions [0,1])" +/request_new_permissions?name=my%20import%20script&basic_permissions=%5B0%2C1%5D +``` + Response: : Some JSON with your access key, which is 64 characters of hex. This will not be valid until the user approves the request in the client ui. ```json title="Example response" @@ -419,6 +428,8 @@ Response: } ``` +The `permits_everything` overrules all the individual permissions and will encompass any new permissions added in future. It is a convenient catch-all for local-only services where you are running things yourself or the user otherwise implicitly trusts you. + ### **GET `/session_key`** { id="session_key" } _Get a new session key._ @@ -455,13 +466,15 @@ Arguments: n/a Response: : 401/403/419 and some error text if the provided access/session key is invalid, otherwise some JSON with basic permission info. + ```json title="Example response" { + "name" : "autotagger", + "permits_everything" : false, "basic_permissions" : [0, 1, 3], "human_description" : "API Permissions (autotagger): add tags to files, import files, search for files: Can search: only autotag this" } ``` - ### **GET `/get_service`** { id="get_service" } @@ -2089,7 +2102,6 @@ If you add `detailed_url_information=true`, a new entry, `detailed_known_urls`, } ``` - ### **GET `/get_files/file`** { id="get_files_file" } _Get a file._ @@ -2105,7 +2117,7 @@ Arguments : * `hash`: (selective, a hexadecimal SHA256 hash for the file) * `download`: (optional, boolean, default `false`) - Only use one of file_id or hash. As with metadata fetching, you may only use the hash argument if you have access to all files. If you are tag-restricted, you will have to use a file_id in the last search you ran. +Only use one of `file_id` or `hash`. As with metadata fetching, you may only use the hash argument if you have access to all files. If you are tag-restricted, you will have to use a file_id in the last search you ran. ``` title="Example request" /get_files/file?file_id=452158 @@ -2119,6 +2131,8 @@ Response: By default, this will set the `Content-Disposition` header to `inline`, which causes a web browser to show the file. If you set `download=true`, it will set it to `attachment`, which triggers the browser to automatically download it (or open the 'save as' dialog) instead. +This stuff supports `Range` requests, so if you want to build a video player, go nuts. + ### **GET `/get_files/thumbnail`** { id="get_files_thumbnail" } _Get a file's thumbnail._ @@ -2133,7 +2147,7 @@ Arguments: * `file_id`: (selective, numerical file id for the file) * `hash`: (selective, a hexadecimal SHA256 hash for the file) - Only use one. As with metadata fetching, you may only use the hash argument if you have access to all files. If you are tag-restricted, you will have to use a file_id in the last search you ran. +Only use one. As with metadata fetching, you may only use the hash argument if you have access to all files. If you are tag-restricted, you will have to use a file_id in the last search you ran. ``` title="Example request" /get_files/thumbnail?file_id=452158 @@ -2155,6 +2169,84 @@ Response: !!! note "Size of Defaults" If you get a 'default' filetype thumbnail like the pdf or hydrus one, you will be pulling the pngs straight from the hydrus/static folder. They will most likely be 200x200 pixels. +### **GET `/get_files/file_path`** { id="get_files_file_path" } + +_Get a local file path._ + +Restricted access: +: YES. Search for Files permission and See Local Paths permission needed. Additional search permission limits may apply. + +Required Headers: n/a + +Arguments : +: + * `file_id`: (selective, numerical file id for the file) + * `hash`: (selective, a hexadecimal SHA256 hash for the file) + +Only use one. As with metadata fetching, you may only use the hash argument if you have access to all files. If you are tag-restricted, you will have to use a file_id in the last search you ran. + +``` title="Example request" +/get_files/file_path?file_id=452158 +``` +``` title="Example request" +/get_files/file_path?hash=7f30c113810985b69014957c93bc25e8eb4cf3355dae36d8b9d011d8b0cf623a +``` + +Response: +: The actual path to the file on the host system. + +``` json title="Example response" +{ + "path" : "D:\hydrus_files\f7f\7f30c113810985b69014957c93bc25e8eb4cf3355dae36d8b9d011d8b0cf623a.jpg" +} +``` + +This will give 404 if the file is not stored locally (which includes if it _should_ exist but is actually missing from the file store). + +### **GET `/get_files/thumbnail_path`** { id="get_files_thumbnail" } + +_Get a local thumbnail path._ + +Restricted access: +: YES. Search for Files permission and See Local Paths permission needed. Additional search permission limits may apply. + +Required Headers: n/a + +Arguments: +: + * `file_id`: (selective, numerical file id for the file) + * `hash`: (selective, a hexadecimal SHA256 hash for the file) + * `include_thumbnail_filetype`: (optional, boolean, defaults to `false`) + +Only use one of `file_id` or `hash`. As with metadata fetching, you may only use the hash argument if you have access to all files. If you are tag-restricted, you will have to use a file_id in the last search you ran. + +``` title="Example request" +/get_files/thumbnail?file_id=452158 +``` +``` title="Example request" +/get_files/thumbnail?hash=7f30c113810985b69014957c93bc25e8eb4cf3355dae36d8b9d011d8b0cf623a&include_thumbnail_filetype=true +``` + +Response: +: The actual path to the thumbnail on the host system. + +``` json title="Example response" +{ + "path" : "D:\hydrus_files\f7f\7f30c113810985b69014957c93bc25e8eb4cf3355dae36d8b9d011d8b0cf623a.thumbnail" +} +``` + +``` json title="Example response with include_thumbnail_filetype=true" +{ + "path" : "C:\hydrus_thumbs\f85\85daaefdaa662761d7cb1b026d7b101e74301be08e50bf09a235794ec8656f79.thumbnail", + "filetype" : "image/png" +} +``` + +All thumbnails in hydrus have the .thumbnail file extension and in content are either jpeg (almost always) or png (to handle transparency). + +This will 400 if the given file type does not have a thumbnail in hydrus, and it will 404 if there should be a thumbnail but one does not exist and cannot be generated from the source file (which probably would mean that the source file was itself Not Found). + ### **GET `/get_files/render`** { id="get_files_render" } _Get an image file as rendered by Hydrus._ diff --git a/docs/old_changelog.html b/docs/old_changelog.html index b371d3c64..7a2f4edd1 100644 --- a/docs/old_changelog.html +++ b/docs/old_changelog.html @@ -34,6 +34,37 @@

changelog