Added software architecture documentation

doc/.readthedocs.yaml

@@ -12,6 +12,8 @@ build:
   tools:
     # Check https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
     python: "3.10"
+  apt_packages:
+    - graphviz
 
 python:
   install:

doc/Makefile

@@ -16,4 +16,5 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
+	@dot -Tsvg "$(SOURCEDIR)/reference/software_architecture.dot" -o "$(SOURCEDIR)/reference/software_architecture.svg";
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/basics/docker.rst

@@ -8,7 +8,7 @@ Currently, only running the (unstable!) ``latest`` release is supported.
 
 
 Preparation
------
+-----------
 
 Fetch the docker image:
 
@@ -17,7 +17,7 @@ Fetch the docker image:
     docker pull ghcr.io/tribler/tribler:latest
 
 Running
------
+-------
 
 Choose a SECRET key for the background communication with Tribler.
 In the following example, we use the key ``changeme`` (don't use this yourself).
@@ -44,7 +44,7 @@ By default, the REST API is bound to localhost inside the container so to
 access the APIs, network needs to be set to host (--net="host").
 
 Stopping
------
+--------
 
 To stop Tribler, you should get the container id of your process and then stop it.
 You can view all active docker containers using ``docker ps`` and you can stop a container id using ``docker stop``.

doc/basics/running.rst

@@ -67,13 +67,14 @@ You can link or copy ``libsodium.dylib`` into the Tribler root directory:
 
 
 Apple Silicon
--------
+-------------
 There are currently no python bindings available to install from pip.
 Therefore you need to build them from source.
 
 To do this, please install openssl and boost first:
 
 .. code-block:: bash
+
     brew install openssl boost boost-build boost-python3
 
 And then follow the `instruction <https://github.com/arvidn/libtorrent/blob/v1.2.18/docs/python_binding.rst>`_.

doc/conf.py

@@ -16,7 +16,13 @@
 # import os
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
+from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+    from sphinx.domains.python import PyObject
 
 # -- Project information -----------------------------------------------------
 
@@ -80,7 +86,7 @@
 exclude_patterns = []
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = None
+pygments_style = 'sphinx'
 
 
 # -- Options for HTML output -------------------------------------------------
@@ -198,3 +204,18 @@
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
+
+
+def skip_illegal_classes(app: Sphinx, what: str, name: str, obj: PyObject, skip: bool, options: list[str]) -> bool:
+    """
+    Sphinx errors out on ``tribler.core.libtorrent.torrentdef.FileDict.__init__``. We get rid of it here.
+    """
+    if what == "class" and name == "tribler.core.libtorrent.torrentdef.FileDict":
+        return True
+    return skip
+
+def setup(sphinx: Sphinx) -> None:
+    """
+    Callback for when Sphinx is setup. We install a workaround for an illegal class here.
+    """
+    sphinx.connect("autoapi-skip-member", skip_illegal_classes)

doc/index.rst

@@ -26,6 +26,12 @@ Table of contents
    building/mac.rst
    building/windows.rst
 
+.. toctree::
+   :maxdepth: 2
+   :caption: Reference:
+
+   reference/software_architecture.rst
+
 
 Search
 ======

doc/reference/software_architecture.dot

@@ -0,0 +1,147 @@
+digraph G {
+  fontname="Helvetica,Arial,sans-serif";
+  node [fontname="Helvetica,Arial,sans-serif"];
+  edge [fontname="Helvetica,Arial,sans-serif"];
+  compound=true;
+  forcelabels=true;
+
+  subgraph cluster_0 {
+    label = "[PYTHON]";
+    style=filled;
+    color=floralwhite;
+
+    subgraph cluster_0_0 {
+      label = "src.tribler.core";
+      style=filled;
+      color=lightgrey;
+      node [style=filled, fillcolor=white, color=black];
+
+      session [label="session.py"];
+
+      db [label="TriblerDatabase", shape=rectangle, style="filled,dashed", fillcolor=gray90];
+      session -> db;
+
+      mds [label="MetadataStore", shape=rectangle, style="filled,dashed", fillcolor=gray90];
+      session -> mds;
+
+      torrentchecker [label="TorrentChecker", shape=rectangle, style="filled,dashed", fillcolor=gray90];
+      session -> torrentchecker;
+
+      configmgr [label="TriblerConfigManager", shape=rectangle];
+      session -> configmgr;
+
+      notifier [label="Notifier", shape=rectangle];
+      session -> notifier;
+
+      dlmgr [label="DownloadManager", shape=rectangle];
+      configmgr -> dlmgr;
+      session -> dlmgr;
+
+      restmgr [label="RESTManager", shape=rectangle];
+      configmgr -> restmgr;
+      session -> restmgr;
+
+      loader [label="IPv8CommunityLoader", shape=rectangle];
+      configmgr -> loader;
+      session -> loader;
+
+      ipv8 [label="IPv8", shape=rectangle];
+      loader -> ipv8;
+      configmgr -> ipv8;
+      session -> ipv8;
+    }
+
+    subgraph cluster_0_1 {
+      label = "src.tribler.core.components";
+      style=filled;
+      fillcolor=lightgrey;
+      color=white;
+      node [style=filled, fillcolor=white, color=black];
+
+      baselauncher [label="BaseLauncher", shape=rectangle];
+      componentlauncher [label="ComponentLauncher", shape=rectangle];
+      {rank=same baselauncher; componentlauncher}
+
+      versioning [label="Versioning", shape=rectangle];
+      componentlauncher -> versioning;
+
+      dbcomp [label="TriblerDatabase", shape=rectangle];
+      componentlauncher -> dbcomp;
+
+      triblerdb [label="TriblerDatabase", shape=rectangle];
+      dbcomp -> triblerdb;
+
+      dhtdisc [label="DHTDiscovery", shape=rectangle];
+      baselauncher -> dhtdisc;
+
+      recomm [label="Recommender", shape=rectangle];
+      baselauncher -> recomm;
+
+      tunnel [label="Tunnel", shape=rectangle];
+      baselauncher -> tunnel;
+
+      mddb [label="MetadataStore", shape=rectangle];
+      dbcomp -> mddb;
+
+      tchecker [label="TorrentChecker", shape=rectangle];
+      componentlauncher -> tchecker;
+
+      contentdiscovery [label="ContentDiscovery", shape=rectangle];
+      baselauncher -> contentdiscovery;
+      dbcomp -> contentdiscovery;
+      tchecker -> contentdiscovery;
+
+      knowledge [label="Knowledge", shape=rectangle];
+      baselauncher -> knowledge;
+      dbcomp -> knowledge;
+
+      rendezvous [label="Rendezvous", shape=rectangle];
+      baselauncher -> rendezvous;
+      dbcomp -> rendezvous;
+    }
+  }
+
+  subgraph cluster_1 {
+    label = "[TYPESCRIPT]\nsrc/tribler/ui";
+    style=filled;
+    color=aliceblue;
+    node [style=filled,fillcolor=white, color=black];
+
+    htmlindex [label="App", shape=rectangle];
+
+    ipv8service [label="IPv8Service", shape=rectangle];
+    triblerservice [label="TriblerService", shape=rectangle];
+
+    error_popup [label="error_popup", shape=rectangle];
+    htmlindex -> error_popup;
+
+    router [label="RouterProvider", shape=rectangle];
+    htmlindex -> router;
+
+    pagedebug [label="Debug", shape=rectangle];
+    router -> pagedebug;
+
+    pagedownloads [label="Downloads", shape=rectangle];
+    router -> pagedownloads;
+
+    pagepopular [label="Popular", shape=rectangle];
+    router -> pagepopular;
+
+    pagesearch [label="Search", shape=rectangle];
+    router -> pagesearch;
+
+    pagesettings [label="Settings", shape=rectangle];
+    router -> pagesettings;
+  }
+
+  start -> session [lhead=cluster_0_0, minlen="2"];
+  start -> htmlindex [lhead=cluster_1, minlen="2", headlabel="                       webbrowser.open_new_tab()", labeldistance=8];
+
+  ipv8 -> baselauncher [style="invis"];
+  loader -> componentlauncher [style="invis"];
+
+  ipv8service -> restmgr [dir="both", style="dotted", arrowhead="vee", arrowtail="vee"];
+  triblerservice -> restmgr [dir="both", style="dotted", arrowhead="vee", arrowtail="vee"];
+
+  start [label="src/run_tribler.py", shape=doubleoctagon, style=filled, fillcolor=floralwhite];
+}

doc/reference/software_architecture.rst

@@ -0,0 +1,86 @@
+Software Architecture
+=====================
+
+This reference provides a high-level overview of Tribler's software architecture.
+
+The main entry point of Tribler is the ``run_tribler.py`` script and it has two main modes.
+The first mode launches Tribler with default settings.
+This causes a new browser tab to open that displays a web page generated with TypeScript.
+The second "headless" mode, does not open a browser tab.
+However, in both modes the core Tribler Python process runs.
+
+An overview of the most important functional components of Tribler is given in the following graph.
+We will provide a short description of each of these components.
+
+.. figure:: software_architecture.svg
+
+Session
+-------
+
+The ``Session`` object (in ``session.py``) manages the core managers of Tribler:
+
+- The ``Notifier`` allows components to publish and subscribe to events.
+  In some special cases, this is preferred over class inheritance or composition.
+  Architectural spaghetti can sometimes be avoided by using ``Notifier`` events.
+- The ``TriblerConfigManager`` manages writing to and reading from user configuration files.
+  This class manages defaults and the regeneration of (partially) missing configurations.
+- The ``DownloadManager`` allows management of torrent downloads.
+  Functionality involves fetching torrent meta information, adding and removing downloads, and changing their anonymity level.
+- The ``RESTManager`` sets up a REST API.
+  The API is used by Tribler's GUI, but can also be used programmatically by users in "headless" mode.
+- ``IPv8`` is used to manage peer-to-peer network overlays.
+  All peer-to-peer traffic is handled by ``IPv8`` with the exception of non-anonymized torrent downloads.
+  Functionality involves decentralized search and sharing popular torrents.
+- The ``IPv8CommunityLoader`` is used to launch all optional parts of Tribler.
+  This includes launching ``IPv8`` overlays and registering REST API endpoints.
+
+The ``TriblerDatabase``, ``MetadataStore`` and the ``TorrentChecker`` are optionally available through the ``Session``.
+If requested, they are launched through the ``IPv8CommunityLoader``.
+
+Components
+----------
+
+The optional components of Tribler are given in ``components.py``.
+They can be divided into two groups: those that require a network overlay (inheriting from ``ComponentLauncher``) and those that do not (inheriting from ``ComponentLauncher``).
+Furthermore, the components can have complex relationships, like only launching after other components.
+
+The following list contains a high-level description of the components:
+
+- The ``DatabaseComponent`` is responsible for the two main databases.
+  The first is the ``MetadataStore``, which stores all torrent and torrent tracker meta data and "health" (seeders and leechers).
+  The second is the ``TriblerDatabase``, which stores torrent tags.
+- The ``TorrentCheckerComponent`` is responsible for retrieving the "health" information of torrents and trackers.
+  This component may contact trackers for random torrents to determine whether they are reachable.
+  The component also periodically updates the health information for known torrents based on tracker and DHT information.
+- The ``VersioningComponent`` is responsible for Tribler version management.
+  Functionality includes upgrading data from local old Tribler versions to new Tribler versions and determining whether a new Tribler version has been published online.
+- The ``TunnelComponent`` allows for anonymization of torrent downloads.
+  This component manages routing downloads over Tor-like circuits, including circuit construction and destruction.
+- The ``DHTDiscoveryComponent`` allows ``IPv8`` to decentrally search for overlays and peers with given public keys.
+- The ``ContentDiscoveryComment`` serves remote user searches and content popularity.
+  Both outgoing and incoming user searches are served through this component.
+- The ``KnowledgeComponent`` handles extraction and search for torrent tags.
+- The ``RendezvousComponent`` keeps track of peers that have been connected to in the past.
+  This is an experimental component that aims to serve as a source of decentral reputation.
+- The ``RecommenderComponent`` keeps track of local user searches and the preferred download for the search query.
+  This is an experimental component that aims to serve as a source for Artificial Intelligence based personalized recommendations.
+
+Web UI
+------
+
+The Web UI serves to view and control the state of the Tribler process.
+Essentially, the Web UI uses three main classes:
+
+- The ``IPv8Service`` binds to the REST API of the ``IPv8`` component in the Tribler process.
+- The ``TriblerService`` binds to the REST API of anything that is not in the ``IPv8`` service.
+- The ``App`` is the React application entrypoint.
+
+The ``App`` contains a component to route information, the ``RouterProvider``, and a means to report errors that have been sent from the Tribler process.
+The architecture closely resembles the information in the GUI.
+Essentially, the ``RouterProvider`` mirrors the visible pages:
+
+- ``src/pages/Debug`` queries the Tribler process for debug information.
+- ``src/pages/Downloads`` includes components to render torrent downloads.
+- ``src/pages/Popular`` queries for popular torrents.
+- ``src/pages/Search`` manages pending searches, combining local and remote results.
+- ``src/pages/Settings`` reads and writes the Tribler configuration (through the REST API).

src/run_tribler.py

@@ -160,7 +160,7 @@ async def mac_event_loop() -> None:
     """
     Consume Mac events on the asyncio main thread.
 
-    WARNING: sendEvent_ can block on some events. In particular, while the tray menu is open.
+    WARNING: ``sendEvent_`` can block on some events. In particular, while the tray menu is open.
     """
     from AppKit import NSApp, NSEventMaskAny
     from Foundation import NSDate, NSDefaultRunLoopMode

src/tribler/core/database/orm_bindings/torrent_metadata.py

@@ -81,7 +81,7 @@ def to_simple_dict(self) -> dict[str, str | bytes | float | None]: ...  # noqa:
 
 def infohash_to_id(infohash: bytes) -> int:
     """
-    This function is used to devise id_ from infohash in deterministic way. Used in FFA channels.
+    This function is used to devise ``id_`` from infohash in deterministic way. Used in FFA channels.
     """
     return abs(unpack(">q", infohash[:8])[0])
 
@@ -107,7 +107,7 @@ def tdef_to_metadata_dict(tdef: TorrentDef) -> dict:
         "title": tdef.get_name_as_unicode()[:300],
         "tags": tags[:200],
         "size": tdef.get_length(),
-        "torrent_date": torrent_date if torrent_date >= EPOCH else EPOCH,
+        "torrent_date": max(torrent_date, EPOCH),
         "tracker_info": tracker_info,
     }
 

src/tribler/core/libtorrent/torrentdef.py

@@ -566,8 +566,7 @@ def get_files_with_length(self, exts: set[str] | None = None) -> list[tuple[Path
         """
         The list of files in the torrent def.
 
-        :param exts: (Optional) list of filename extensions (without leading .)
-        to search for.
+        :param exts: (Optional) list of filename extensions (without leading .) to search for.
         :return: A list of filenames.
         """
         videofiles = []