Fix : System Message: ERROR/3 (<string>, line 18) Unknown directive t…

…ype toctree. #537

datagrepper/app.py

@@ -24,18 +24,17 @@
 
 import arrow
 import datanommer.models as dm
-import docutils
-import docutils.examples
 import flask
-import jinja2
 import markupsafe
 import pygal
 import pygments
 import pygments.formatters
 import pygments.lexers
+from bs4 import BeautifulSoup as bs4
 from flask import Flask
 from flask_healthz import HealthError, healthz
 from pkg_resources import get_distribution
+from sphinx.cmd.build import main as sphinx_main
 from werkzeug.exceptions import BadRequest
 
 from datagrepper.util import (
@@ -131,74 +130,28 @@ def remove_session(exc):
     dm.session.remove()
 
 
-def modify_rst(rst):
-    """Downgrade some of our rst directives if docutils is too old."""
-
-    try:
-        # The rst features we need were introduced in this version
-        minimum = [0, 9]
-        version = map(int, docutils.__version__.split("."))
-
-        # If we're at or later than that version, no need to downgrade
-        if version >= minimum:
-            return rst
-    except Exception:
-        # If there was some error parsing or comparing versions, run the
-        # substitutions just to be safe.
-        pass
-
-    # Otherwise, make code-blocks into just literal blocks.
-    substitutions = {
-        ".. code-block:: javascript": "::",
-    }
-    for old, new in substitutions.items():
-        rst = rst.replace(old, new)
-
-    return rst
-
-
-def modify_html(html):
-    """Perform style substitutions where docutils doesn't do what we want."""
-
-    substitutions = {
-        '<tt class="docutils literal">': "<code>",
-        "</tt>": "</code>",
-    }
-    for old, new in substitutions.items():
-        html = html.replace(old, new)
-
-    return html
-
-
 def preload_docs(endpoint):
     """Utility to load an RST file and turn it into fancy HTML."""
 
     here = os.path.dirname(os.path.abspath(__file__))
     default = os.path.join(here, "docs")
     directory = app.config.get("DATAGREPPER_DOC_PATH", default)
-    fname = os.path.join(directory, endpoint + ".rst")
+    htmldocs = os.path.join(directory, "_build/html/")
+    args = ["-b", "html", "-c", default, default, htmldocs]
+    sphinx_main(args)
+    fname = os.path.join(htmldocs, endpoint + ".html")
     with codecs.open(fname, "r", "utf-8") as f:
-        rst = f.read()
-
-    rst = modify_rst(rst)
-
-    api_docs = docutils.examples.html_body(rst)
+        api_docs = f.read()
 
-    api_docs = modify_html(api_docs)
+    soup = bs4(api_docs, "html.parser")
+    api_docs = soup.find("body").decode_contents()
 
     api_docs = markupsafe.Markup(api_docs)
     return api_docs
 
 
-htmldocs = dict.fromkeys(["index", "reference", "widget", "charts"])
-for key in htmldocs:
-    htmldocs[key] = preload_docs(key)
-
-
 def load_docs(request):
-    URL = app.config.get("DATAGREPPER_BASE_URL", request.url_root)
-    docs = htmldocs[request.endpoint]
-    docs = jinja2.Template(docs).render(URL=URL)
+    docs = preload_docs(request.endpoint)
     return markupsafe.Markup(docs)
 
 
@@ -248,6 +201,11 @@ def widget():
     return flask.render_template("index.html", docs=load_docs(flask.request))
 
 
+@app.route("/contributing.html")
+def contributing():
+    return flask.render_template("index.html", docs=load_docs(flask.request))
+
+
 @app.route("/raw", methods=["GET"], strict_slashes=False)
 @app.route("/v2/search", methods=["GET"], strict_slashes=False)
 def raw():

datagrepper/docs/conf.py

@@ -20,6 +20,9 @@
 
 
 # -- General configuration ------------------------------------------------
+rst_epilog = """
+.. |URL| replace:: ``https://apps.fedoraproject.org/datagrepper/``
+"""
 
 # If your documentation needs a minimal Sphinx version, state it here.
 #

datagrepper/docs/index.rst

@@ -45,27 +45,33 @@ Requesting all messages in the last 2 days
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 datagrepper takes time arguments in `seconds`. So, we need to convert two days
-to 172,800 seconds first. Then, we can use HTTPie_ to get the JSON payload::
+to 172,800 seconds first. Then, we can use HTTPie_ to get the JSON payload
 
-   http get {{URL}}v2/search delta==172800
+.. parsed-literal::
+
+   http get \ |URL|\ v2/search delta==172800
 
 
 Requesting all messages in fixed time range
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To get messages in fixed absolute time range, we can use a date/time string,
-for example the common ISO 8601 format::
+for example the common ISO 8601 format
+
+.. parsed-literal::
 
-   http get {{URL}}v2/search end==2021-06-25T06:11:40+00:00 start==2021-06-25T06:11:39+00:00
+   http get \ |URL|\ v2/search end==2021-06-25T06:11:40+00:00 start==2021-06-25T06:11:39+00:00
 
 
 Paging results
 --------------
 
 The previous example is a large JSON response that's too big to read through.
-Limit the number of results to make it more digestable::
+Limit the number of results to make it more digestable
+
+.. parsed-literal::
 
-   http get {{URL}}v2/search delta==172800 rows_per_page==1
+   http get \ |URL|\ v2/search delta==172800 rows_per_page==1
 
 .. code-block:: javascript
 
@@ -104,12 +110,14 @@ Notice a few things.
 #. **Pagination**: ``rows_per_page`` shows the rows per page, its sibling value
    ``page`` is pointer to "page" of data you are on
 
-Use this command to get to the next page::
+Use this command to get to the next page
 
-   http get {{URL}}v2/search \
-      delta==172800 \
-      rows_per_page==1 \
-      page==2
+.. parsed-literal::
+
+   http get \ |URL|\ v2/search \
+   delta==172800 \
+   rows_per_page==1 \
+   page==2
 
 .. code-block:: javascript
 
@@ -146,55 +154,66 @@ set it to ``asc`` for ascending order (i.e. oldest to newest).
 Only Bodhi messages (OR wiki)
 -----------------------------
 
-Specify a ``category`` to limit your message to one kind of topic::
+Specify a ``category`` to limit your message to one kind of topic
+
+.. parsed-literal::
 
-   http get {{URL}}v2/search \
-      delta==172800 \
-      category==bodhi
+   http get \ |URL|\ v2/search \
+   delta==172800 \
+   category==bodhi
 
 Here, ``category`` is singular but comes back in the ``arguments`` dict as
 *categories* (plural)! You can specify multiple categories and messages that
-match *either* category will return. They are ``OR``'d together::
+match *either* category will return. They are ``OR``'d together
+
+.. parsed-literal::
 
-   http get {{URL}}v2/search \
-      delta==172800 \
-      category==bodhi \
-      category==wiki
+   http get \ |URL|\ v2/search \
+   delta==172800 \
+   category==bodhi \
+   category==wiki
 
 Messages for specific users and packages
 ----------------------------------------
 
-Search for events relating to multiple users with this query::
+Search for events relating to multiple users with this query
 
-   http get {{URL}}v2/search \
-      delta==172800 \
-      user==toshio \
-      user==pingou
+.. parsed-literal::
 
-Same for packages::
+   http get \ |URL|\ v2/search \
+   delta==172800 \
+   user==toshio \
+   user==pingou
 
-   http get {{URL}}v2/search \
-      delta==172800 \
-      package==nethack
+Same for packages
 
+.. parsed-literal::
+
+   http get \ |URL|\ v2/search \
+   delta==172800 \
+   package==nethack
 
 Excluding data
 --------------
 
 For each positive filter, there is a corresponding *negative filter*. If you
-want to query all messages **except for Koji messages**, use this query::
+want to query all messages **except for Koji messages**, use this query
+
+.. parsed-literal::
 
-   http get {{URL}}v2/search \
-      delta==172800 \
-      not_category==buildsys
+   http get \ |URL|\ v2/search \
+   delta==172800 \
+   not_category==buildsys
 
 Positive and negative filters are combinable. This query returns all messages
-except for user ``toshio``'s *Ask Fedora* activity::
+except for user ``toshio``'s *Ask Fedora* activity
+
+.. parsed-literal::
 
-   http get {{URL}}v2/search \
-      delta==172800 \
-      user==toshio \
-      not_category==askbot
+   http get \ |URL|\ v2/search \
+   delta==172800 \
+   user==toshio \
+   not_category==askbot
 
 
 Putting it all together (CNF)
@@ -204,14 +223,16 @@ Multiple ``category``, ``user``, and ``package`` filters are merged together in
 a way that looks like `Conjunctive Normal Form`_ (CNF).
 
 The following query returns all messages from the past two days where
-*(category==bodhi OR category==wiki) AND (user==toshio OR user==pingou)*::
-
-   http get {{URL}}v2/search \
-      delta==172800 \
-      category==bodhi \
-      category==wiki \
-      user==toshio \
-      user==pingou
+*(category==bodhi OR category==wiki) AND (user==toshio OR user==pingou)*
+
+.. parsed-literal::
+
+   http get \ |URL|\ v2/search \
+   delta==172800 \
+   category==bodhi \
+   category==wiki \
+   user==toshio \
+   user==pingou
 
 
 Get help

datagrepper/static/css/main.css

@@ -12,3 +12,20 @@ div {
 .marketing {
   margin-top: 3em;
 }
+
+a.headerlink {
+    visibility: hidden;
+}
+
+h1:hover > a.headerlink,
+h2:hover > a.headerlink,
+h3:hover > a.headerlink,
+h4:hover > a.headerlink,
+h5:hover > a.headerlink,
+h6:hover > a.headerlink,
+dt:hover > a.headerlink,
+caption:hover > a.headerlink,
+p.caption:hover > a.headerlink,
+div.code-block-caption:hover > a.headerlink {
+    visibility: visible;
+}

devel/ansible/roles/datagrepper/templates/fedora-messaging.toml.j2

@@ -30,6 +30,7 @@ routing_keys = ["#"]  # Set this to the specific topics you are interested in.
 
 [consumer_config]
 datanommer_sqlalchemy_url = "postgresql://datanommer:datanommer@localhost/messages"
+alembic_ini = "../datanommer.models/alembic.ini"
 
 [qos]
 prefetch_size = 0