Add theme column

* Add support for filtering by theme, country_code and asns
* Add support for setting slug of finding
* Add support for retrieving findings by slug

ooniapi/common/src/common/alembic/versions/8e7ecea5c2f5_themes_column.py

@@ -0,0 +1,63 @@
+"""themes column
+
+Revision ID: 8e7ecea5c2f5
+Revises: a037e908f3a0
+Create Date: 2024-10-11 12:03:07.662702
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "8e7ecea5c2f5"
+down_revision: Union[str, None] = "a037e908f3a0"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def convert_json_to_array(column_name: str):
+    op.add_column("oonifinding", sa.Column(f"{column_name}_tmp", sa.ARRAY(sa.String)))
+    op.execute(
+        f"""
+        UPDATE oonifinding
+        SET {column_name}_tmp = ARRAY(SELECT json_array_elements_text({column_name}))
+    """
+    )
+    op.drop_column("oonifinding", column_name)
+    op.alter_column("oonifinding", f"{column_name}_tmp", new_column_name=column_name)
+
+
+def convert_array_to_json(column_name: str):
+    op.add_column("oonifinding", sa.Column(f"{column_name}_tmp", sa.JSONB))
+    op.execute(
+        f"""
+        UPDATE oonifinding
+        SET {column_name}_tmp = to_json({column_name})
+    """
+    )
+    op.drop_column("oonifinding", column_name)
+    op.alter_column("oonifinding", f"{column_name}_tmp", new_column_name=column_name)
+
+
+def upgrade() -> None:
+    op.add_column("oonifinding", sa.Column("themes", sa.ARRAY(sa.String())))
+    convert_json_to_array("country_codes")
+    convert_json_to_array("asns")
+    convert_json_to_array("domains")
+    convert_json_to_array("tags")
+    convert_json_to_array("links")
+    convert_json_to_array("test_names")
+
+
+def downgrade() -> None:
+    op.drop_column("oonifinding", "themes")
+    convert_array_to_json("country_codes")
+    convert_array_to_json("asns")
+    convert_array_to_json("domains")
+    convert_array_to_json("tags")
+    convert_array_to_json("links")
+    convert_array_to_json("test_names")

ooniapi/common/src/common/alembic/versions/a037e908f3a0_init_oonifindings.py

@@ -5,14 +5,15 @@
 Create Date: 2024-07-17 16:45:25.752551
 
 """
+
 from typing import Sequence, Union
 
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision: str = 'a037e908f3a0'
+revision: str = "a037e908f3a0"
 down_revision: Union[str, None] = "c9119c05cf42"
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None

ooniapi/services/oonifindings/src/oonifindings/models.py

@@ -15,7 +15,7 @@ class OONIFinding(Base):
     # TODO(decfox): this is nullable for now. We should
     # make this a non-nullable field eventually and have an endpoint
     # where we can query findings using the finding_slug.
-    finding_slug: Mapped[str] = mapped_column(String, nullable=True)
+    finding_slug: Mapped[str] = mapped_column(String, nullable=False, unique=True)
 
     create_time: Mapped[datetime] = mapped_column(UtcDateTime())
     update_time: Mapped[datetime] = mapped_column(UtcDateTime())
@@ -35,6 +35,7 @@ class OONIFinding(Base):
     country_codes: Mapped[List[str]] = mapped_column(nullable=True)
     asns: Mapped[List[str]] = mapped_column(nullable=True)
     domains: Mapped[List[str]] = mapped_column(nullable=True)
+    themes: Mapped[List[str]] = mapped_column(nullable=True)
     tags: Mapped[List[str]] = mapped_column(nullable=True)
     links: Mapped[List[str]] = mapped_column(nullable=True)
     test_names: Mapped[List[str]] = mapped_column(nullable=True)