feat: add voilib management dashboard

backend/bin/docker-entrypoint.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+echo "Running docker-entrypoint initialization script"
+
+# run migrations on sqlite database
+alembic upgrade head
+
+# create, if needed, the admin user
+voilib-management --create-admin
+
+# run the CMD passed as command-line arguments
+exec "$@"

backend/dockerfile

@@ -24,3 +24,7 @@ COPY . .
 # allow installing development dependencies to run tests
 ARG INSTALL_DEV=false
 RUN bash -c "if [ $INSTALL_DEV == 'true' ] ; then pip install -e .[dev] ; else pip install . ; fi"
+
+RUN chmod +x "/backend/bin/docker-entrypoint.sh"
+ENTRYPOINT ["/backend/bin/docker-entrypoint.sh"]
+CMD ["uvicorn", "src.voilib.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "80"]

backend/requirements.txt

@@ -1,9 +1,13 @@
 aiosqlite==0.19.0
 alembic==1.12.0
+altair==5.1.1
 anyio==4.0.0
 async-timeout==4.0.3
+attrs==23.1.0
 av==10.0.0
 bcrypt==4.0.1
+blinker==1.6.2
+cachetools==5.3.1
 certifi==2023.7.22
 cffi==1.15.1
 charset-normalizer==3.2.0
@@ -23,6 +27,8 @@ faster-whisper==0.7.1
 filelock==3.12.3
 flatbuffers==23.5.26
 fsspec==2023.9.0
+gitdb==4.0.10
+GitPython==3.1.34
 greenlet==2.0.2
 grpcio==1.57.0
 grpcio-tools==1.57.0
@@ -36,12 +42,17 @@ huggingface-hub==0.16.4
 humanfriendly==10.0
 hyperframe==6.0.1
 idna==3.4
+importlib-metadata==6.8.0
 itsdangerous==2.1.2
 Jinja2==3.1.2
 joblib==1.3.2
+jsonschema==4.19.0
+jsonschema-specifications==2023.7.1
 lit==16.0.6
 Mako==1.2.4
+markdown-it-py==3.0.0
 MarkupSafe==2.1.3
+mdurl==0.1.2
 mpmath==1.3.0
 networkx==3.1
 nltk==3.8.1
@@ -61,22 +72,32 @@ onnxruntime==1.15.1
 orjson==3.9.5
 ormar==0.12.2
 packaging==23.1
+pandas==2.1.0
 passlib==1.7.4
-Pillow==10.0.0
+Pillow==9.5.0
 portalocker==2.7.0
 protobuf==4.24.2
+pyarrow==13.0.0
 pyasn1==0.5.0
 pycparser==2.21
 pydantic==1.10.8
+pydeck==0.8.0
+Pygments==2.16.1
+Pympler==1.0.1
 python-dateutil==2.8.2
 python-dotenv==1.0.0
 python-jose==3.3.0
 python-multipart==0.0.6
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
 PyYAML==6.0.1
 qdrant-client==1.4.0
 redis==5.0.0
+referencing==0.30.2
 regex==2023.8.8
 requests==2.31.0
+rich==13.5.2
+rpds-py==0.10.0
 rq==1.15.1
 rsa==4.9
 safetensors==0.3.3
@@ -85,22 +106,33 @@ scipy==1.11.2
 sentence-transformers==2.2.2
 sentencepiece==0.1.99
 six==1.16.0
+smmap==5.0.0
 sniffio==1.3.0
 SQLAlchemy==1.4.41
 starlette==0.27.0
+streamlit==1.26.0
 sympy==1.12
+tenacity==8.2.3
 threadpoolctl==3.2.0
 tokenizers==0.13.3
+toml==0.10.2
+toolz==0.12.0
 torch==2.0.1
 torchvision==0.15.2
+tornado==6.3.3
 tqdm==4.66.1
 transformers==4.32.1
 triton==2.0.0
 typing_extensions==4.7.1
+tzdata==2023.3
+tzlocal==4.3.1
 ujson==5.8.0
 urllib3==1.26.16
 uvicorn==0.23.2
 uvloop==0.17.0
+validators==0.22.0
+watchdog==3.0.0
 watchfiles==0.20.0
 websockets==11.0.3
 xmltodict==0.13.0
+zipp==3.16.2

backend/setup.cfg

@@ -30,6 +30,7 @@ install_requires =
     sentence-transformers>=2.2.2,<3
     qdrant-client>=1.1.6,<2
     faster-whisper>=0.7.1,<1
+    streamlit>=1.26.0,<2
 
 [options.packages.find]
 where=src

backend/src/voilib/management/pages/1_🔑-Login.py

@@ -0,0 +1,60 @@
+# Copyright (c) 2022-2023 Pablo González Carrizo (unmonoqueteclea)
+# All rights reserved.
+
+import asyncio
+import typing
+from datetime import timedelta
+
+import streamlit as st
+
+from voilib import auth
+
+st.set_page_config(page_title="Voilib", page_icon="🎧")
+st.title("🔑 Login")
+
+SHOW_LOGIN_FORM = True
+USERNAME_KEY = "logged_user_username"
+TOKEN_KEY = "logged_user_token"
+
+
+async def _login(username: str, password: str) -> typing.Optional[str]:
+    user = await auth.authenticate_user(username, password)
+    if user:
+        return auth.create_access_token(
+            data={"sub": user.username},  # type: ignore
+            expires_delta=timedelta(minutes=auth.ACCESS_TOKEN_EXPIRE_MINUTES),
+        )
+
+
+async def main():
+    global SHOW_LOGIN_FORM
+    if USERNAME_KEY in st.session_state and TOKEN_KEY in st.session_state:
+        SHOW_LOGIN_FORM = False
+    if SHOW_LOGIN_FORM:
+        with st.form(key="login"):
+            username = st.text_input("Username")
+            password = st.text_input("Password", type="password")
+            clicked = st.form_submit_button("Login", use_container_width=True)
+        if clicked:
+            token = await _login(username, password)
+            if token:
+                st.session_state[USERNAME_KEY] = username
+                st.session_state[TOKEN_KEY] = token
+                SHOW_LOGIN_FORM = False
+                st.experimental_rerun()
+            else:
+                st.error("Invalid credentials. Please, try again")
+    else:
+        username = st.session_state[USERNAME_KEY]
+        st.info(f"""👤  Already logged in as **{username}**""")
+        logout = st.button("Logout", use_container_width=True)
+        if logout:
+            del st.session_state[USERNAME_KEY]
+            del st.session_state[TOKEN_KEY]
+            SHOW_LOGIN_FORM = True
+            st.experimental_rerun()
+
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    loop.run_until_complete(main())

backend/src/voilib/management/pages/2_📈-Stats.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2022-2023 Pablo González Carrizo (unmonoqueteclea)
+# All rights reserved.
+
+import asyncio
+
+import pandas as pd
+import streamlit as st
+
+from voilib.management import utils
+from voilib.models import analytics
+
+
+async def main():
+    st.set_page_config(page_title="Voilib", page_icon="🎧")
+    st.title("📈 Stats")
+    authenticated = utils.login_message(st.session_state)
+
+    if authenticated:
+        tab_last, tab_graphs = st.tabs(["Last queries", "Queries per day"])
+        with tab_last:
+            st.write("Last 20 queries performed by Voilib users")
+            qs = await analytics.Query.objects.order_by("-created_at").limit(20).all()
+            markdown_queries = ""
+            for query in qs:
+                date = query.created_at.strftime("%Y-%m-%d, %H:%M:%S")  # type: ignore
+                markdown_queries += f"\n - `{date}` {query.text}"
+            if len(qs) == 0:
+                st.write("⚠️ No queries yet!")
+            st.markdown(markdown_queries)
+        with tab_graphs:
+            qs = await analytics.Query.objects.order_by("-created_at").values(
+                fields=["created_at", "text"]
+            )
+            df = pd.DataFrame(qs)
+            if df.shape[0] == 0:
+                st.write("⚠️ No queries yet!")
+            else:
+                df["created_at"] = pd.to_datetime(df["created_at"]).dt.date
+                st.bar_chart(data=df.created_at.value_counts())
+        refresh = st.button("Refresh", use_container_width=True)
+        if refresh:
+            st.experimental_rerun()
+
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    loop.run_until_complete(main())

backend/src/voilib/management/pages/3_🔈-Media.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2022-2023 Pablo González Carrizo (unmonoqueteclea)
+# All rights reserved.
+
+import asyncio
+
+import streamlit as st
+from voilib import collection, models, routers, settings
+from voilib.management import utils as m_utils
+
+
+async def add_channel():
+    st.header("Add new podcast")
+    with st.form("my_form"):
+        st.markdown(
+            """Write below the RSS feed url from a podcast and click `ADD`
+            to include it in the database. """
+        )
+        st.markdown(
+            """After adding a new channel, you should
+
+            """
+        )
+        channel_url = st.text_input("Channel RSS feed url")
+        add_click = st.form_submit_button("Add channel", use_container_width=True)
+        if add_click:
+            with st.spinner("⌛ Adding new channel... Please, wait."):
+                _, ch = await collection.get_or_create_channel(channel_url)
+                settings.queue.enqueue(
+                    collection.update_channel, ch, job_timeout="600m"
+                )
+            st.success(
+                f"""Channel "{ch.title}" correctly added to the
+                database.  Its episodes are being updated in a
+                background task. This process can take a few minutes."""
+            )
+
+
+async def podcasts_and_episodes():
+    st.header("Podcasts and episodes")
+
+    col1, col2, col3 = st.columns(3)
+    col1.metric("Channels", await models.Channel.objects.count())
+    col2.metric(
+        "Transcribed episodes",
+        await models.Episode.objects.filter(transcribed=True).count(),
+    )
+    col3.metric(
+        "Indexed episodes",
+        await models.Episode.objects.filter(embeddings=True).count(),
+    )
+    with st.spinner("⌛ Loading channels..."):
+        for ch in (await routers.analytics._media()).channels:
+            with st.expander(
+                f"**{ch.title}**. Indexed {ch.available_episodes}/{ch.total_episodes}"
+            ):
+                st.image(ch.image)
+                st.markdown(ch.description)
+
+
+async def main():
+    st.set_page_config(page_title="Voilib", page_icon="🎧")
+    st.title("📻 Media")
+    authenticated = m_utils.login_message(st.session_state)
+    if authenticated:
+        await add_channel()
+        st.divider()
+        await podcasts_and_episodes()
+
+
+if __name__ == "__main__":
+    loop = asyncio.new_event_loop()
+    loop.run_until_complete(main())