diff --git a/.taskcluster.yml b/.taskcluster.yml index 50f198ea..ef7e14fd 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -28,22 +28,11 @@ tasks: then: ${event.pull_request.base.repo.clone_url} else: ${event.repository.clone_url} - codecov_secret: - codecov-fuzzmanager - - pypi_secret: - pypi-fuzzmanager - project_name: FuzzManager matrix: language: python - secrets: - - type: env - secret: project/fuzzing/codecov-fuzzmanager - name: CODECOV_TOKEN - key: token script: - bash - '-xec' @@ -96,24 +85,6 @@ tasks: npm run test; npm run production; npm run codecov; - - name: PyPI upload - version: "3.9" - env: - TOXENV: pypi - script: - - tox - when: - release: true - all_passed: true - secrets: - - type: env - secret: project/fuzzing/pypi-fuzzmanager - name: TWINE_USERNAME - key: username - - type: env - secret: project/fuzzing/pypi-fuzzmanager - name: TWINE_PASSWORD - key: password in: $if: > @@ -148,88 +119,9 @@ tasks: - queue:create-task:highest:proj-fuzzing/ci - queue:create-task:highest:proj-fuzzing/ci-* - queue:scheduler-id:taskcluster-github - - secrets:get:project/fuzzing/${codecov_secret} - - secrets:get:project/fuzzing/${pypi_secret} metadata: name: ${project_name} CI decision description: Schedule CI tasks for ${project_name} owner: '${user}@users.noreply.github.com' source: ${http_repo}/raw/${fetch_rev}/.taskcluster.yml - - taskId: {$eval: as_slugid("docker")} - taskGroupId: ${task_group} - provisionerId: proj-fuzzing - workerType: ci - dependencies: [] - created: {$fromNow: ''} - deadline: {$fromNow: '1 hour'} - payload: - image: - namespace: project.fuzzing.orion.orion-builder.master - path: public/orion-builder.tar.zst - type: indexed-image - maxRunTime: 3600 - capabilities: - privileged: true - env: - LOAD_DEPS: "0" - GIT_REPOSITORY: ${http_repo} - GIT_REVISION: ${fetch_rev} - BUILD_TOOL: podman - DOCKERFILE: Dockerfile - IMAGE_NAME: mozillasecurity/fuzzmanager - ARCHIVE_PATH: /image.tar - command: - - sh - - -c - - uname -a && exec build - artifacts: - public/fuzzmanager.tar.zst: - expires: {$fromNow: '6 months'} - path: /image.tar.zst - type: file - scopes: - - docker-worker:capability:privileged - metadata: - name: FuzzManager Docker build - description: FuzzManager Docker build - owner: '${user}@users.noreply.github.com' - source: ${http_repo}/raw/${fetch_rev}/.taskcluster.yml - - $if: 'tasks_for in ["github-push"] && fetch_ref == "refs/heads/master"' - then: - taskId: {$eval: as_slugid("docker_push")} - taskGroupId: ${task_group} - provisionerId: proj-fuzzing - workerType: ci - dependencies: - - {$eval: as_slugid("docker")} - created: {$fromNow: ''} - deadline: {$fromNow: '1 hour'} - payload: - capabilities: - privileged: true - image: - namespace: project.fuzzing.orion.orion-builder.master - path: public/orion-builder.tar.zst - type: indexed-image - maxRunTime: 3600 - features: - taskclusterProxy: true - env: - TASK_ID: {$eval: as_slugid("docker")} - TASKCLUSTER_SECRET: "project/fuzzing/docker-hub" - GIT_REPOSITORY: ${http_repo} - GIT_REVISION: ${fetch_rev} - SERVICE_NAME: fuzzmanager - command: - - sh - - -c - - uname -a && exec push - scopes: - - docker-worker:capability:privileged - - secrets:get:project/fuzzing/docker-hub - metadata: - name: FuzzManager Docker push - description: FuzzManager Docker push - owner: '${user}@users.noreply.github.com' - source: ${http_repo}/raw/${fetch_rev}/.taskcluster.yml else: [] diff --git a/server/crashmanager/management/commands/create_test_data.py b/server/crashmanager/management/commands/create_test_data.py index c52e2323..d873ab2a 100644 --- a/server/crashmanager/management/commands/create_test_data.py +++ b/server/crashmanager/management/commands/create_test_data.py @@ -1,7 +1,10 @@ +import json from base64 import b64encode +from copy import deepcopy from datetime import timedelta from pathlib import Path from random import choice, randint +from shutil import rmtree from subprocess import run from uuid import uuid4 @@ -11,7 +14,13 @@ from notifications.models import Notification from notifications.signals import notify -from covmanager.models import Collection, CollectionFile, Repository +from covmanager.models import ( + Collection, + CollectionFile, + Report, + ReportConfiguration, + Repository, +) from crashmanager.models import ( OS, Bucket, @@ -28,6 +37,7 @@ Tool, User, ) +from FTB.CoverageHelper import calculate_summary_fields, merge_coverage_data from taskmanager.models import Pool, Task @@ -167,15 +177,123 @@ def slug(): return b64encode(raw, altchars=b"-_").decode()[:22] +COV_DATA_1 = { + "children": { + "a": { + "children": { + "test.c": { + "coverage": [-1, -1, 1, 1, 1, 1, 0, -1, 1], + "name": "test.c", + } + }, + "name": "a", + }, + "b": { + "children": { + "testb.c": { + "coverage": [-1, -1, 1, 1, 1], + "name": "testb.c", + } + }, + "name": "b", + }, + "main.c": { + "coverage": [-1, -1, -1, -1, -1, -1, -1, 1, 1, 1, -1, 1, -1, 2], + "name": "main.c", + }, + }, +} +calculate_summary_fields(COV_DATA_1) +COV_DATA_2 = { + "children": { + "a": { + "children": { + "test.c": { + "coverage": [-1, -1, 1, 1, 1, 1, 1, -1, 1], + "name": "test.c", + } + }, + "name": "a", + }, + "b": { + "children": { + "testb.c": { + "coverage": [-1, -1, 1, 1, 1], + "name": "testb.c", + } + }, + "name": "b", + }, + "main.c": { + "coverage": [-1, -1, -1, -1, -1, -1, -1, 2, 1, 1, 1, -1, 1, -1, 2], + "name": "main.c", + }, + }, +} +calculate_summary_fields(COV_DATA_2) +COV_DATA_3 = { + "children": { + "a": { + "children": { + "test.c": { + "coverage": [-1, -1, 1, 1, 1, 1, 1, -1, 1], + "name": "test.c", + } + }, + "name": "a", + }, + "b": { + "children": { + "testb.c": { + "coverage": [-1, -1, 1, 1, 1], + "name": "testb.c", + } + }, + "name": "b", + }, + "main.c": { + "coverage": [-1, -1, -1, -1, -1, -1, -1, 1, 2, 1, 1, -1, 1, -1, 2], + "name": "main.c", + }, + }, +} +calculate_summary_fields(COV_DATA_3) + + class Command(BaseCommand): help = "Create test data for UI testing (requires empty DB)" def add_arguments(self, parser): parser.add_argument("user") + parser.add_argument( + "--delete-all-existing-data-forever", + action="store_true", + help="Delete existing data without warning", + ) def handle(self, *_args, **options): user = User.objects.get(user__username=options["user"]) + if options["delete_all_existing_data_forever"]: + Bucket.objects.all().delete() + BucketWatch.objects.all().delete() + Bug.objects.all().delete() + BugProvider.objects.all().delete() + BugzillaTemplate.objects.all().delete() + Client.objects.all().delete() + Collection.objects.all().delete() + CollectionFile.objects.all().delete() + Notification.objects.all().delete() + OS.objects.all().delete() + Platform.objects.all().delete() + Pool.objects.all().delete() + Product.objects.all().delete() + Repository.objects.all().delete() + Task.objects.all().delete() + Tool.objects.all().delete() + if Path("/data/repos/cov-example").exists(): + rmtree("/data/repos/cov-example") + # assert that the DB is empty. we don't want real data mixed with test data try: assert not Bucket.objects.exists() @@ -231,32 +349,18 @@ def handle(self, *_args, **options): Path("/data/coverage/coverage").mkdir(parents=True, exist_ok=True) Path( "/data/coverage/coverage/0f87343595e8908a2989ca7716f71502bf288c72.coverage" - ).write_text( - '{"children":{"a":{"children":{"test.c":{"coverage":[-1,-1,1,1,1,1,0,-1,1],' - '"name":"test.c","linesTotal":6,"linesCovered":5,"linesMissed":1,"coverageP' - 'ercent":83.33}},"name":"a","linesTotal":6,"linesCovered":5,"linesMissed":1' - ',"coveragePercent":83.33},"b":{"children":{"testb.c":{"coverage":[-1,-1,1,' - '1,1],"name":"testb.c","linesTotal":3,"linesCovered":3,"linesMissed":0,"cov' - 'eragePercent":100.0}},"name":"b","linesTotal":3,"linesCovered":3,"linesMis' - 'sed":0,"coveragePercent":100.0},"main.c":{"coverage":[-1,-1,-1,-1,-1,-1,-1' - ',2,2,1,-1,1,-1,2],"name":"main.c","linesTotal":5,"linesCovered":5,"linesMi' - 'ssed":0,"coveragePercent":100.0}},"name":null,"linesTotal":14,"linesCovere' - 'd":13,"linesMissed":1,"coveragePercent":92.86}' - ) + ).write_text(json.dumps(COV_DATA_1)) Path( "/data/coverage/coverage/58721097e138f17549dc129d7dcc44a0adebe218.coverage" - ).write_text( - '{"children":{"a":{"children":{"test.c":{"coverage":[-1,-1,1,1,1,1,1,-1,1],' - '"name":"test.c","linesTotal":6,"linesCovered":6,"linesMissed":0,"coverageP' - 'ercent":100.0}},"name":"a","linesTotal":6,"linesCovered":6,"linesMissed":0' - ',"coveragePercent":100.0},"b":{"children":{"testb.c":{"coverage":[-1,-1,1,' - '1,1],"name":"testb.c","linesTotal":3,"linesCovered":3,"linesMissed":0,"cov' - 'eragePercent":100.0}},"name":"b","linesTotal":3,"linesCovered":3,"linesMis' - 'sed":0,"coveragePercent":100.0},"main.c":{"coverage":[-1,-1,-1,-1,-1,-1,-1' - ',2,2,1,1,-1,1,-1,2],"name":"main.c","linesTotal":6,"linesCovered":6,"lines' - 'Missed":0,"coveragePercent":100.0}},"name":null,"linesTotal":15,"linesCove' - 'red":15,"linesMissed":0,"coveragePercent":100.0}' - ) + ).write_text(json.dumps(COV_DATA_2)) + Path( + "/data/coverage/coverage/68721097e138f17549dc129d7dcc44a0adebe218.coverage" + ).write_text(json.dumps(COV_DATA_3)) + merged_covdata = deepcopy(COV_DATA_3) + merge_coverage_data(merged_covdata, COV_DATA_2) + Path( + "/data/coverage/coverage/78721097e138f17549dc129d7dcc44a0adebe218.coverage" + ).write_text(json.dumps(merged_covdata)) call_command( "setup_repository", "cov-example", @@ -265,7 +369,7 @@ def handle(self, *_args, **options): ) repo = Repository.objects.get(name="cov-example") - Collection.objects.create( + initial_cov = Collection.objects.create( branch="main", client=client, coverage=CollectionFile.objects.create( @@ -287,6 +391,48 @@ def handle(self, *_args, **options): repository=repo, revision="adab95a85e138f792631f19d939dfd1102197acc", ) + Collection.objects.create( + branch="main", + client=client, + coverage=CollectionFile.objects.create( + file="coverage/68721097e138f17549dc129d7dcc44a0adebe218.coverage" + ), + created="2024-11-21T22:48:41Z", + description="update #2", + repository=repo, + revision="adab95a85e138f792631f19d939dfd1102197acc", + ) + agg = Collection.objects.create( + branch="main", + client=client, + coverage=CollectionFile.objects.create( + file="coverage/78721097e138f17549dc129d7dcc44a0adebe218.coverage" + ), + created="2024-11-21T22:48:41Z", + description="Test aggregate", + repository=repo, + revision="adab95a85e138f792631f19d939dfd1102197acc", + ) + Report.objects.create( + coverage=initial_cov, + ) + Report.objects.create( + coverage=agg, + public=True, + ) + rc1 = ReportConfiguration.objects.create( + description="Test configuration", + directives="+:*\n-:a/*", + public=True, + repository=repo, + ) + ReportConfiguration.objects.create( + description="A", + directives="+:a/*", + logical_parent=rc1, + public=True, + repository=repo, + ) # create 250 unbucketed crashes for _ in range(250): diff --git a/server/frontend/package-lock.json b/server/frontend/package-lock.json index cafe076c..49245c9e 100644 --- a/server/frontend/package-lock.json +++ b/server/frontend/package-lock.json @@ -9528,9 +9528,9 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", "funding": [ { "type": "github",