feat: use db models for feature util instead of hardcoded params [WIP] (

#130)

* add initial models + migrations

* add indices

* fix typo + make migrations

* remove unnecessary models

* run lint

* consolidate migrations

* lint

* add more comments

* squashed: address pr comments, removed redundant constraint and change field type

add sql migration comment

allow empty array for override ids

use strings for override keys instead

fix-typo

* change overrides back to ids from strings

* add auto-inc PK to variants

* address pr comments: removed redundant constraint and change field type

* allow empty array for override ids

* use db instead of hard-coded params for Feature util

* refactor + implement feedback

* run lint

* turn ttl back to 5 minutes

* initial seed migration for feature already in codebase

* update field name

* update fieldname

* change override from string back to ids

* use new unconstrained name for variants

* use better array field for UX

* fix type

* update tests to use new Feature

* remove override ids in migration

docker-compose.yml

@@ -35,8 +35,8 @@ services:
   timescale:
     image: timescale/timescaledb-ha:pg14-latest
     environment:
-      - POSTGRES_USER=timescale
-      - POSTGRES_PASSWORD=timescale
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
       - POSTGRES_HOST_AUTH_METHOD=trust
     volumes:
       - type: tmpfs

setup.py

@@ -48,5 +48,7 @@
         "sqlalchemy==1.*",
         "ijson==3.*",
         "codecov-ribs",
+        "cachetools",
+        "django-better-admin-arrayfield",
     ],
 )

shared/django_apps/dummy_settings.py

@@ -2,13 +2,15 @@
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent
+from .db_settings import *
 
 ALLOWED_HOSTS = []
 
 # Install apps so that you can make migrations for them
 INSTALLED_APPS = [
     "shared.django_apps.pg_telemetry",
     "shared.django_apps.ts_telemetry",
+    "shared.django_apps.rollouts",
 ]
 
 MIDDLEWARE = []
@@ -34,8 +36,8 @@
     "timeseries": {
         "ENGINE": "django.db.backends.postgresql",
         "NAME": "POSTGRES_USER",
-        "USER": "timescale",
-        "PASSWORD": "timescale",
+        "USER": "postgres",
+        "PASSWORD": "postgres",
         "HOST": "timescale",
         "PORT": 5432,
     },

shared/django_apps/rollouts/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class RolloutsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "shared.django_apps.rollouts"

shared/django_apps/rollouts/migrations/0001_initial.py

@@ -0,0 +1,99 @@
+# Generated by Django 4.2.7 on 2024-02-26 18:57
+
+import django.db.models.deletion
+import django_better_admin_arrayfield.models.fields
+from django.db import migrations, models
+
+import shared.django_apps.rollouts.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    # BEGIN;
+    # --
+    # -- Create model FeatureFlag
+    # --
+    # CREATE TABLE "feature_flags" ("name" varchar(200) NOT NULL PRIMARY KEY, "proportion" decimal NOT NULL, "salt" varchar(32) NOT NULL);
+    # --
+    # -- Create model FeatureFlagVariant
+    # --
+    # CREATE TABLE "feature_flag_variants" ("variant_id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(200) NOT NULL, "proportion" decimal NOT NULL, "value" text NOT NULL CHECK ((JSON_VALID("value") OR "value" IS NULL)), "override_owner_ids" integer[] NOT NULL, "override_repo_ids" integer[] NOT NULL, "feature_flag_id" varchar(200) NOT NULL REFERENCES "feature_flags" ("name") DEFERRABLE INITIALLY DEFERRED);
+    # CREATE INDEX "feature_flag_variants_feature_flag_id_fa3a4c02" ON "feature_flag_variants" ("feature_flag_id");
+    # CREATE INDEX "feature_fla_feature_15a078_idx" ON "feature_flag_variants" ("feature_flag_id");
+    # COMMIT;
+
+    operations = [
+        migrations.CreateModel(
+            name="FeatureFlag",
+            fields=[
+                (
+                    "name",
+                    models.CharField(max_length=200, primary_key=True, serialize=False),
+                ),
+                (
+                    "proportion",
+                    models.DecimalField(decimal_places=3, default=0, max_digits=4),
+                ),
+                (
+                    "salt",
+                    models.CharField(
+                        default=shared.django_apps.rollouts.models.default_random_salt,
+                        max_length=32,
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "feature_flags",
+            },
+        ),
+        migrations.CreateModel(
+            name="FeatureFlagVariant",
+            fields=[
+                ("variant_id", models.AutoField(primary_key=True, serialize=False)),
+                ("name", models.CharField(max_length=200)),
+                (
+                    "proportion",
+                    models.DecimalField(decimal_places=3, default=0, max_digits=4),
+                ),
+                ("value", models.JSONField(default=False)),
+                (
+                    "override_owner_ids",
+                    django_better_admin_arrayfield.models.fields.ArrayField(
+                        base_field=models.IntegerField(),
+                        blank=True,
+                        default=list,
+                        size=None,
+                    ),
+                ),
+                (
+                    "override_repo_ids",
+                    django_better_admin_arrayfield.models.fields.ArrayField(
+                        base_field=models.IntegerField(),
+                        blank=True,
+                        default=list,
+                        size=None,
+                    ),
+                ),
+                (
+                    "feature_flag",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="variants",
+                        to="rollouts.featureflag",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "feature_flag_variants",
+                "indexes": [
+                    models.Index(
+                        fields=["feature_flag"], name="feature_fla_feature_15a078_idx"
+                    )
+                ],
+            },
+        ),
+    ]

shared/django_apps/rollouts/migrations/0002_auto_20240226_1858.py

@@ -0,0 +1,37 @@
+# Generated by Django 4.2.7 on 2024-02-26 18:58
+
+from django.db import migrations
+
+
+def seed_initial_features(apps, schema_editor):
+    FeatureFlag = apps.get_model("rollouts", "FeatureFlag")
+    FeatureFlagVariant = apps.get_model("rollouts", "FeatureFlagVariant")
+
+    list_repos_generator = FeatureFlag.objects.create(
+        name="list_repos_generator", proportion=0.0
+    )
+    FeatureFlagVariant.objects.create(
+        name="enabled",
+        feature_flag=list_repos_generator,
+        proportion=1.0,
+        value=True,
+    )
+
+    use_label_index_in_report_processing = FeatureFlag.objects.create(
+        name="use_label_index_in_report_processing", proportion=0.0
+    )
+    FeatureFlagVariant.objects.create(
+        name="enabled",
+        feature_flag=use_label_index_in_report_processing,
+        proportion=1.0,
+        value=True,
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("rollouts", "0001_initial"),
+    ]
+
+    operations = [migrations.RunPython(seed_initial_features)]

shared/django_apps/rollouts/models.py

@@ -0,0 +1,63 @@
+from random import choice
+
+from django.db import models
+from django_better_admin_arrayfield.models.fields import ArrayField
+
+
+# TODO: move to utils
+def default_random_salt():
+    chars = []
+    ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    for _ in range(16):
+        chars.append(choice(ALPHABET))
+    return "".join(chars)
+
+
+class FeatureFlag(models.Model):
+    """
+    Represents a feature and its rollout parameters (see shared/rollouts/__init__.py). A
+    default salt will be created if one is not provided.
+    """
+
+    name = models.CharField(max_length=200, primary_key=True)
+    proportion = models.DecimalField(default=0, decimal_places=3, max_digits=4)
+    salt = models.CharField(max_length=32, default=default_random_salt)
+
+    class Meta:
+        db_table = "feature_flags"
+
+    def __str__(self):
+        return self.name
+
+
+class FeatureFlagVariant(models.Model):
+    """
+    Represents a variant of the feature being rolled out and the proportion of
+    the test population it should be rolled out to (see shared/rollouts/__init__.py).
+    The proportion should be a float between 0 and 1. A proportion of 0.5 means 50% of
+    the test population should receive this variant. Ensure that for any `FeatureFlag`,
+    the proportions of the corresponding `FeatureFlagVariant`s sum to 1.
+    """
+
+    variant_id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=200)
+    feature_flag = models.ForeignKey(
+        "FeatureFlag", on_delete=models.CASCADE, related_name="variants"
+    )
+    proportion = models.DecimalField(default=0, decimal_places=3, max_digits=4)
+    value = models.JSONField(default=False)
+
+    # Weak foreign keys to Owner and Respository models respectively
+    override_owner_ids = ArrayField(
+        base_field=models.IntegerField(), default=list, blank=True
+    )
+    override_repo_ids = ArrayField(
+        base_field=models.IntegerField(), default=list, blank=True
+    )
+
+    class Meta:
+        db_table = "feature_flag_variants"
+        indexes = [models.Index(fields=["feature_flag"])]
+
+    def __str__(self):
+        return self.feature_flag.__str__() + ": " + self.name