Bump nanoid from 3.3.7 to 3.3.8 in /server/frontend

.taskcluster.yml

@@ -28,22 +28,11 @@ tasks:
       then: ${event.pull_request.base.repo.clone_url}
       else: ${event.repository.clone_url}
 
-    codecov_secret:
-      codecov-fuzzmanager
-
-    pypi_secret:
-      pypi-fuzzmanager
-
     project_name:
       FuzzManager
 
     matrix:
       language: python
-      secrets:
-        - type: env
-          secret: project/fuzzing/codecov-fuzzmanager
-          name: CODECOV_TOKEN
-          key: token
       script:
         - bash
         - '-xec'
@@ -96,24 +85,6 @@ tasks:
                 npm run test;
                 npm run production;
                 npm run codecov;
-          - name: PyPI upload
-            version: "3.9"
-            env:
-              TOXENV: pypi
-            script:
-              - tox
-            when:
-              release: true
-              all_passed: true
-            secrets:
-              - type: env
-                secret: project/fuzzing/pypi-fuzzmanager
-                name: TWINE_USERNAME
-                key: username
-              - type: env
-                secret: project/fuzzing/pypi-fuzzmanager
-                name: TWINE_PASSWORD
-                key: password
 
   in:
     $if: >
@@ -148,88 +119,9 @@ tasks:
           - queue:create-task:highest:proj-fuzzing/ci
           - queue:create-task:highest:proj-fuzzing/ci-*
           - queue:scheduler-id:taskcluster-github
-          - secrets:get:project/fuzzing/${codecov_secret}
-          - secrets:get:project/fuzzing/${pypi_secret}
         metadata:
           name: ${project_name} CI decision
           description: Schedule CI tasks for ${project_name}
           owner: '${user}@users.noreply.github.com'
           source: ${http_repo}/raw/${fetch_rev}/.taskcluster.yml
-      - taskId: {$eval: as_slugid("docker")}
-        taskGroupId: ${task_group}
-        provisionerId: proj-fuzzing
-        workerType: ci
-        dependencies: []
-        created: {$fromNow: ''}
-        deadline: {$fromNow: '1 hour'}
-        payload:
-          image:
-            namespace: project.fuzzing.orion.orion-builder.master
-            path: public/orion-builder.tar.zst
-            type: indexed-image
-          maxRunTime: 3600
-          capabilities:
-            privileged: true
-          env:
-            LOAD_DEPS: "0"
-            GIT_REPOSITORY: ${http_repo}
-            GIT_REVISION: ${fetch_rev}
-            BUILD_TOOL: podman
-            DOCKERFILE: Dockerfile
-            IMAGE_NAME: mozillasecurity/fuzzmanager
-            ARCHIVE_PATH: /image.tar
-          command:
-            - sh
-            - -c
-            - uname -a && exec build
-          artifacts:
-            public/fuzzmanager.tar.zst:
-              expires: {$fromNow: '6 months'}
-              path: /image.tar.zst
-              type: file
-        scopes:
-          - docker-worker:capability:privileged
-        metadata:
-          name: FuzzManager Docker build
-          description: FuzzManager Docker build
-          owner: '${user}@users.noreply.github.com'
-          source: ${http_repo}/raw/${fetch_rev}/.taskcluster.yml
-      - $if: 'tasks_for in ["github-push"] && fetch_ref == "refs/heads/master"'
-        then:
-          taskId: {$eval: as_slugid("docker_push")}
-          taskGroupId: ${task_group}
-          provisionerId: proj-fuzzing
-          workerType: ci
-          dependencies:
-            - {$eval: as_slugid("docker")}
-          created: {$fromNow: ''}
-          deadline: {$fromNow: '1 hour'}
-          payload:
-            capabilities:
-              privileged: true
-            image:
-              namespace: project.fuzzing.orion.orion-builder.master
-              path: public/orion-builder.tar.zst
-              type: indexed-image
-            maxRunTime: 3600
-            features:
-              taskclusterProxy: true
-            env:
-              TASK_ID: {$eval: as_slugid("docker")}
-              TASKCLUSTER_SECRET: "project/fuzzing/docker-hub"
-              GIT_REPOSITORY: ${http_repo}
-              GIT_REVISION: ${fetch_rev}
-              SERVICE_NAME: fuzzmanager
-            command:
-              - sh
-              - -c
-              - uname -a && exec push
-          scopes:
-            - docker-worker:capability:privileged
-            - secrets:get:project/fuzzing/docker-hub
-          metadata:
-            name: FuzzManager Docker push
-            description: FuzzManager Docker push
-            owner: '${user}@users.noreply.github.com'
-            source: ${http_repo}/raw/${fetch_rev}/.taskcluster.yml
     else: []

server/crashmanager/management/commands/create_test_data.py

@@ -1,7 +1,10 @@
+import json
 from base64 import b64encode
+from copy import deepcopy
 from datetime import timedelta
 from pathlib import Path
 from random import choice, randint
+from shutil import rmtree
 from subprocess import run
 from uuid import uuid4
 
@@ -11,7 +14,13 @@
 from notifications.models import Notification
 from notifications.signals import notify
 
-from covmanager.models import Collection, CollectionFile, Repository
+from covmanager.models import (
+    Collection,
+    CollectionFile,
+    Report,
+    ReportConfiguration,
+    Repository,
+)
 from crashmanager.models import (
     OS,
     Bucket,
@@ -28,6 +37,7 @@
     Tool,
     User,
 )
+from FTB.CoverageHelper import calculate_summary_fields, merge_coverage_data
 from taskmanager.models import Pool, Task
 
 
@@ -167,15 +177,123 @@ def slug():
     return b64encode(raw, altchars=b"-_").decode()[:22]
 
 
+COV_DATA_1 = {
+    "children": {
+        "a": {
+            "children": {
+                "test.c": {
+                    "coverage": [-1, -1, 1, 1, 1, 1, 0, -1, 1],
+                    "name": "test.c",
+                }
+            },
+            "name": "a",
+        },
+        "b": {
+            "children": {
+                "testb.c": {
+                    "coverage": [-1, -1, 1, 1, 1],
+                    "name": "testb.c",
+                }
+            },
+            "name": "b",
+        },
+        "main.c": {
+            "coverage": [-1, -1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1, -1, 2],
+            "name": "main.c",
+        },
+    },
+}
+calculate_summary_fields(COV_DATA_1)
+COV_DATA_2 = {
+    "children": {
+        "a": {
+            "children": {
+                "test.c": {
+                    "coverage": [-1, -1, 1, 1, 1, 1, 1, -1, 1],
+                    "name": "test.c",
+                }
+            },
+            "name": "a",
+        },
+        "b": {
+            "children": {
+                "testb.c": {
+                    "coverage": [-1, -1, 1, 1, 1],
+                    "name": "testb.c",
+                }
+            },
+            "name": "b",
+        },
+        "main.c": {
+            "coverage": [-1, -1, -1, -1, -1, -1, -1, 2, 1, 1, 1, -1, 1, -1, 2],
+            "name": "main.c",
+        },
+    },
+}
+calculate_summary_fields(COV_DATA_2)
+COV_DATA_3 = {
+    "children": {
+        "a": {
+            "children": {
+                "test.c": {
+                    "coverage": [-1, -1, 1, 1, 1, 1, 1, -1, 1],
+                    "name": "test.c",
+                }
+            },
+            "name": "a",
+        },
+        "b": {
+            "children": {
+                "testb.c": {
+                    "coverage": [-1, -1, 1, 1, 1],
+                    "name": "testb.c",
+                }
+            },
+            "name": "b",
+        },
+        "main.c": {
+            "coverage": [-1, -1, -1, -1, -1, -1, -1, 1, 2, 1, 1, -1, 1, -1, 2],
+            "name": "main.c",
+        },
+    },
+}
+calculate_summary_fields(COV_DATA_3)
+
+
 class Command(BaseCommand):
     help = "Create test data for UI testing (requires empty DB)"
 
     def add_arguments(self, parser):
         parser.add_argument("user")
+        parser.add_argument(
+            "--delete-all-existing-data-forever",
+            action="store_true",
+            help="Delete existing data without warning",
+        )
 
     def handle(self, *_args, **options):
         user = User.objects.get(user__username=options["user"])
 
+        if options["delete_all_existing_data_forever"]:
+            Bucket.objects.all().delete()
+            BucketWatch.objects.all().delete()
+            Bug.objects.all().delete()
+            BugProvider.objects.all().delete()
+            BugzillaTemplate.objects.all().delete()
+            Client.objects.all().delete()
+            Collection.objects.all().delete()
+            CollectionFile.objects.all().delete()
+            Notification.objects.all().delete()
+            OS.objects.all().delete()
+            Platform.objects.all().delete()
+            Pool.objects.all().delete()
+            Product.objects.all().delete()
+            Repository.objects.all().delete()
+            Task.objects.all().delete()
+            Tool.objects.all().delete()
+            if Path("/data/repos/cov-example").exists():
+                rmtree("/data/repos/cov-example")
+
         # assert that the DB is empty. we don't want real data mixed with test data
         try:
             assert not Bucket.objects.exists()
@@ -231,32 +349,18 @@ def handle(self, *_args, **options):
         Path("/data/coverage/coverage").mkdir(parents=True, exist_ok=True)
         Path(
             "/data/coverage/coverage/0f87343595e8908a2989ca7716f71502bf288c72.coverage"
-        ).write_text(
-            '{"children":{"a":{"children":{"test.c":{"coverage":[-1,-1,1,1,1,1,0,-1,1],'
-            '"name":"test.c","linesTotal":6,"linesCovered":5,"linesMissed":1,"coverageP'
-            'ercent":83.33}},"name":"a","linesTotal":6,"linesCovered":5,"linesMissed":1'
-            ',"coveragePercent":83.33},"b":{"children":{"testb.c":{"coverage":[-1,-1,1,'
-            '1,1],"name":"testb.c","linesTotal":3,"linesCovered":3,"linesMissed":0,"cov'
-            'eragePercent":100.0}},"name":"b","linesTotal":3,"linesCovered":3,"linesMis'
-            'sed":0,"coveragePercent":100.0},"main.c":{"coverage":[-1,-1,-1,-1,-1,-1,-1'
-            ',2,2,1,-1,1,-1,2],"name":"main.c","linesTotal":5,"linesCovered":5,"linesMi'
-            'ssed":0,"coveragePercent":100.0}},"name":null,"linesTotal":14,"linesCovere'
-            'd":13,"linesMissed":1,"coveragePercent":92.86}'
-        )
+        ).write_text(json.dumps(COV_DATA_1))
         Path(
             "/data/coverage/coverage/58721097e138f17549dc129d7dcc44a0adebe218.coverage"
-        ).write_text(
-            '{"children":{"a":{"children":{"test.c":{"coverage":[-1,-1,1,1,1,1,1,-1,1],'
-            '"name":"test.c","linesTotal":6,"linesCovered":6,"linesMissed":0,"coverageP'
-            'ercent":100.0}},"name":"a","linesTotal":6,"linesCovered":6,"linesMissed":0'
-            ',"coveragePercent":100.0},"b":{"children":{"testb.c":{"coverage":[-1,-1,1,'
-            '1,1],"name":"testb.c","linesTotal":3,"linesCovered":3,"linesMissed":0,"cov'
-            'eragePercent":100.0}},"name":"b","linesTotal":3,"linesCovered":3,"linesMis'
-            'sed":0,"coveragePercent":100.0},"main.c":{"coverage":[-1,-1,-1,-1,-1,-1,-1'
-            ',2,2,1,1,-1,1,-1,2],"name":"main.c","linesTotal":6,"linesCovered":6,"lines'
-            'Missed":0,"coveragePercent":100.0}},"name":null,"linesTotal":15,"linesCove'
-            'red":15,"linesMissed":0,"coveragePercent":100.0}'
-        )
+        ).write_text(json.dumps(COV_DATA_2))
+        Path(
+            "/data/coverage/coverage/68721097e138f17549dc129d7dcc44a0adebe218.coverage"
+        ).write_text(json.dumps(COV_DATA_3))
+        merged_covdata = deepcopy(COV_DATA_3)
+        merge_coverage_data(merged_covdata, COV_DATA_2)
+        Path(
+            "/data/coverage/coverage/78721097e138f17549dc129d7dcc44a0adebe218.coverage"
+        ).write_text(json.dumps(merged_covdata))
         call_command(
             "setup_repository",
             "cov-example",
@@ -265,7 +369,7 @@ def handle(self, *_args, **options):
         )
 
         repo = Repository.objects.get(name="cov-example")
-        Collection.objects.create(
+        initial_cov = Collection.objects.create(
             branch="main",
             client=client,
             coverage=CollectionFile.objects.create(
@@ -287,6 +391,48 @@ def handle(self, *_args, **options):
             repository=repo,
             revision="adab95a85e138f792631f19d939dfd1102197acc",
         )
+        Collection.objects.create(
+            branch="main",
+            client=client,
+            coverage=CollectionFile.objects.create(
+                file="coverage/68721097e138f17549dc129d7dcc44a0adebe218.coverage"
+            ),
+            created="2024-11-21T22:48:41Z",
+            description="update #2",
+            repository=repo,
+            revision="adab95a85e138f792631f19d939dfd1102197acc",
+        )
+        agg = Collection.objects.create(
+            branch="main",
+            client=client,
+            coverage=CollectionFile.objects.create(
+                file="coverage/78721097e138f17549dc129d7dcc44a0adebe218.coverage"
+            ),
+            created="2024-11-21T22:48:41Z",
+            description="Test aggregate",
+            repository=repo,
+            revision="adab95a85e138f792631f19d939dfd1102197acc",
+        )
+        Report.objects.create(
+            coverage=initial_cov,
+        )
+        Report.objects.create(
+            coverage=agg,
+            public=True,
+        )
+        rc1 = ReportConfiguration.objects.create(
+            description="Test configuration",
+            directives="+:*\n-:a/*",
+            public=True,
+            repository=repo,
+        )
+        ReportConfiguration.objects.create(
+            description="A",
+            directives="+:a/*",
+            logical_parent=rc1,
+            public=True,
+            repository=repo,
+        )
 
         # create 250 unbucketed crashes
         for _ in range(250):