diff --git a/.github/workflows/api-pull-request.yml b/.github/workflows/api-pull-request.yml index 3da5fed6af..8143ca7f41 100644 --- a/.github/workflows/api-pull-request.yml +++ b/.github/workflows/api-pull-request.yml @@ -1,4 +1,4 @@ -name: "API - Pull Request" +name: "API - Pull Request" on: push: @@ -148,7 +148,7 @@ jobs: working-directory: ./api if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' run: | - poetry run pytest -n auto --cov=./src/backend --cov-report=xml src/backend + poetry run pytest --cov=./src/backend --cov-report=xml src/backend - name: Upload coverage reports to Codecov if: steps.are-non-ignored-files-changed.outputs.any_changed == 'true' uses: codecov/codecov-action@v5 diff --git a/.gitignore b/.gitignore index f115325102..4c588f5698 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ build/ /dist/ *.egg-info/ */__pycache__/*.pyc +.idea/ # Session Session.vim diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f871bf6de1..2fcb4d4c6f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -97,12 +97,13 @@ repos: - id: safety name: safety description: "Safety is a tool that checks your installed dependencies for known security vulnerabilities" - entry: bash -c 'safety check --ignore 70612' + entry: bash -c 'safety check --ignore 70612,66963' language: system - id: vulture name: vulture description: "Vulture finds unused code in Python programs." entry: bash -c 'vulture --exclude "contrib" --min-confidence 100 .' + exclude: 'api/src/backend/' language: system files: '.*\.py' diff --git a/api/poetry.lock b/api/poetry.lock index ef5e2bfe26..c03e9e4b18 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -439,13 +439,13 @@ azure-core = ">=1.26.2,<2.0.0" [[package]] name = "azure-mgmt-cosmosdb" -version = "9.6.0" +version = "9.7.0" description = "Microsoft Azure Cosmos DB Management Client Library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "azure_mgmt_cosmosdb-9.6.0-py3-none-any.whl", hash = "sha256:02b4108867de58e0b89a206ee7b7588b439e1f6fef2377ce1979b803a0d02d5a"}, - {file = "azure_mgmt_cosmosdb-9.6.0.tar.gz", hash = "sha256:667c7d8a8f542b0e7972e63274af536ad985187e24a6cc2e3c8eef35560881fc"}, + {file = "azure_mgmt_cosmosdb-9.7.0-py3-none-any.whl", hash = "sha256:be735a554d16995c8cefe413e62119985f8fabae1cb45a6f6ad2c3958bed14da"}, + {file = "azure_mgmt_cosmosdb-9.7.0.tar.gz", hash = "sha256:b5072d319f11953d8f12e22459aded1912d5f27e442e1d8b49596a85005410a1"}, ] [package.dependencies] @@ -537,6 +537,22 @@ azure-mgmt-core = ">=1.3.2" isodate = ">=0.6.1" typing-extensions = ">=4.6.0" +[[package]] +name = "azure-mgmt-search" +version = "9.1.0" +description = "Microsoft Azure Search Management Client Library for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "azure-mgmt-search-9.1.0.tar.gz", hash = "sha256:53bc6eeadb0974d21f120bb21bb5e6827df6d650e17347460fd83e2d68883599"}, + {file = "azure_mgmt_search-9.1.0-py3-none-any.whl", hash = "sha256:488ff81477e980e2b7abf0b857387c74ebbad419e6f6126044e3e6fad2da72b6"}, +] + +[package.dependencies] +azure-common = ">=1.1,<2.0" +azure-mgmt-core = ">=1.3.2,<2.0.0" +isodate = ">=0.6.1,<1.0.0" + [[package]] name = "azure-mgmt-security" version = "7.0.0" @@ -686,17 +702,17 @@ files = [ [[package]] name = "boto3" -version = "1.35.60" +version = "1.35.66" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.60-py3-none-any.whl", hash = "sha256:a34d28de1a1f6ca6ec3edd05c26db16e422293d8f9dcd94f308059a434596753"}, - {file = "boto3-1.35.60.tar.gz", hash = "sha256:e573504c67c3e438fd4b0222119ed1a73b644c78eb3b6dee0b36a6c70ecf7677"}, + {file = "boto3-1.35.66-py3-none-any.whl", hash = "sha256:09a610f8cf4d3c22d4ca69c1f89079e3a1c82805ce94fa0eb4ecdd4d2ba6c4bc"}, + {file = "boto3-1.35.66.tar.gz", hash = "sha256:c392b9168b65e9c23483eaccb5b68d1f960232d7f967a1e00a045ba065ce050d"}, ] [package.dependencies] -botocore = ">=1.35.60,<1.36.0" +botocore = ">=1.35.66,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -705,13 +721,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.60" +version = "1.35.69" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.60-py3-none-any.whl", hash = "sha256:ddccfc39a0a55ac0321191a36d29c2ea9be2c96ceefb3928dd3c91c79c494d50"}, - {file = "botocore-1.35.60.tar.gz", hash = "sha256:378f53037d817bed2c04a006b7319745e664030182211429c924647273b29bc9"}, + {file = "botocore-1.35.69-py3-none-any.whl", hash = "sha256:cad8d9305f873404eee4b197d84e60a40975d43cbe1ab63abe893420ddfe6e3c"}, + {file = "botocore-1.35.69.tar.gz", hash = "sha256:f9f23dd76fb247d9b0e8d411d2995e6f847fc451c026f1e58e300f815b0b36eb"}, ] [package.dependencies] @@ -1891,13 +1907,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.153.0" +version = "2.154.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_python_client-2.153.0-py2.py3-none-any.whl", hash = "sha256:6ff13bbfa92a57972e33ec3808e18309e5981b8ca1300e5da23bf2b4d6947384"}, - {file = "google_api_python_client-2.153.0.tar.gz", hash = "sha256:35cce8647f9c163fc04fb4d811fc91aae51954a2bdd74918decbe0e65d791dd2"}, + {file = "google_api_python_client-2.154.0-py2.py3-none-any.whl", hash = "sha256:a521bbbb2ec0ba9d6f307cdd64ed6e21eeac372d1bd7493a4ab5022941f784ad"}, + {file = "google_api_python_client-2.154.0.tar.gz", hash = "sha256:1b420062e03bfcaa1c79e2e00a612d29a6a934151ceb3d272fe150a656dc8f17"}, ] [package.dependencies] @@ -3275,7 +3291,7 @@ files = [ [[package]] name = "prowler" -version = "4.6.0" +version = "5.0.0" description = "Prowler is an Open Source security tool to perform AWS, GCP and Azure security best practices assessments, audits, incident response, continuous monitoring, hardening and forensics readiness. It contains hundreds of controls covering CIS, NIST 800, NIST CSF, CISA, RBI, FedRAMP, PCI-DSS, GDPR, HIPAA, FFIEC, SOC2, GXP, AWS Well-Architected Framework Security Pillar, AWS Foundational Technical Review (FTR), ENS (Spanish National Security Scheme) and your custom security frameworks." optional = false python-versions = ">=3.9,<3.13" @@ -3292,26 +3308,27 @@ azure-mgmt-authorization = "4.0.0" azure-mgmt-compute = "33.0.0" azure-mgmt-containerregistry = "10.3.0" azure-mgmt-containerservice = "33.0.0" -azure-mgmt-cosmosdb = "9.6.0" +azure-mgmt-cosmosdb = "9.7.0" azure-mgmt-keyvault = "10.3.1" azure-mgmt-monitor = "6.0.2" azure-mgmt-network = "28.0.0" azure-mgmt-rdbms = "10.1.0" azure-mgmt-resource = "23.2.0" +azure-mgmt-search = "9.1.0" azure-mgmt-security = "7.0.0" azure-mgmt-sql = "3.0.1" azure-mgmt-storage = "21.2.1" azure-mgmt-subscription = "3.1.1" azure-mgmt-web = "7.3.1" azure-storage-blob = "12.24.0" -boto3 = "1.35.60" -botocore = "1.35.60" +boto3 = "1.35.66" +botocore = "1.35.69" colorama = "0.4.6" cryptography = "43.0.1" dash = "2.18.2" dash-bootstrap-components = "1.6.0" detect-secrets = "1.5.0" -google-api-python-client = "2.153.0" +google-api-python-client = "2.154.0" google-auth-httplib2 = ">=0.1,<0.3" jsonschema = "4.23.0" kubernetes = "31.0.0" @@ -3325,7 +3342,7 @@ python-dateutil = "^2.9.0.post0" pytz = "2024.2" schema = "0.7.7" shodan = "1.31.0" -slack-sdk = "3.33.3" +slack-sdk = "3.33.4" tabulate = "0.9.0" tzlocal = "5.2" @@ -3333,7 +3350,7 @@ tzlocal = "5.2" type = "git" url = "https://github.com/prowler-cloud/prowler.git" reference = "master" -resolved_reference = "8be83fc632445cd25eeb90ed20257716b673cead" +resolved_reference = "9c383baff309d868b37934b694df9aacba397fad" [[package]] name = "psutil" @@ -4458,13 +4475,13 @@ files = [ [[package]] name = "slack-sdk" -version = "3.33.3" +version = "3.33.4" description = "The Slack API Platform SDK for Python" optional = false python-versions = ">=3.6" files = [ - {file = "slack_sdk-3.33.3-py2.py3-none-any.whl", hash = "sha256:0515fb93cd03b18de61f876a8304c4c3cef4dd3c2a3bad62d7394d2eb5a3c8e6"}, - {file = "slack_sdk-3.33.3.tar.gz", hash = "sha256:4cc44c9ffe4bb28a01fbe3264c2f466c783b893a4eca62026ab845ec7c176ff1"}, + {file = "slack_sdk-3.33.4-py2.py3-none-any.whl", hash = "sha256:9f30cb3c9c07b441c49d53fc27f9f1837ad1592a7e9d4ca431f53cdad8826cc6"}, + {file = "slack_sdk-3.33.4.tar.gz", hash = "sha256:5e109847f6b6a22d227609226ba4ed936109dc00675bddeb7e0bee502d3ee7e0"}, ] [package.extras] diff --git a/api/src/backend/api/filters.py b/api/src/backend/api/filters.py index 2ed175c5d7..7a3e850430 100644 --- a/api/src/backend/api/filters.py +++ b/api/src/backend/api/filters.py @@ -4,47 +4,48 @@ from django.db.models import Q from django_filters.rest_framework import ( BaseInFilter, - FilterSet, BooleanFilter, CharFilter, - UUIDFilter, - DateFilter, ChoiceFilter, + DateFilter, + FilterSet, + UUIDFilter, ) from rest_framework_json_api.django_filters.backends import DjangoFilterBackend from rest_framework_json_api.serializers import ValidationError from api.db_utils import ( - ProviderEnumField, FindingDeltaEnumField, - StatusEnumField, - SeverityEnumField, InvitationStateEnumField, + ProviderEnumField, + SeverityEnumField, + StatusEnumField, ) from api.models import ( - User, + ComplianceOverview, + Finding, + Invitation, Membership, Provider, ProviderGroup, + ProviderSecret, Resource, ResourceTag, Scan, - Task, - StateChoices, - Finding, + ScanSummary, SeverityChoices, + StateChoices, StatusChoices, - ProviderSecret, - Invitation, - ComplianceOverview, + Task, + User, ) from api.rls import Tenant from api.uuid_utils import ( datetime_to_uuid7, - uuid7_start, + transform_into_uuid7, uuid7_end, uuid7_range, - transform_into_uuid7, + uuid7_start, ) from api.v1.serializers import TaskBase @@ -57,6 +58,13 @@ def to_html(self, _request, _queryset, _view): """ return None + def get_filterset_class(self, view, queryset=None): + # Check if the view has 'get_filterset_class' method + if hasattr(view, "get_filterset_class"): + return view.get_filterset_class() + # Fallback to the default implementation + return super().get_filterset_class(view, queryset) + class UUIDInFilter(BaseInFilter, UUIDFilter): pass @@ -482,3 +490,28 @@ class Meta: "version": ["exact", "icontains"], "region": ["exact", "icontains", "in"], } + + +class ScanSummaryFilter(FilterSet): + inserted_at = DateFilter(field_name="inserted_at", lookup_expr="date") + provider_id = UUIDFilter(field_name="scan__provider__id", lookup_expr="exact") + provider_type = ChoiceFilter( + field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices + ) + provider_type__in = ChoiceInFilter( + field_name="scan__provider__provider", choices=Provider.ProviderChoices.choices + ) + region = CharFilter(field_name="region") + muted_findings = BooleanFilter(method="filter_muted_findings") + + def filter_muted_findings(self, queryset, name, value): + if not value: + return queryset.exclude(muted__gt=0) + return queryset + + class Meta: + model = ScanSummary + fields = { + "inserted_at": ["date", "gte", "lte"], + "region": ["exact", "icontains", "in"], + } diff --git a/api/src/backend/api/migrations/0001_initial.py b/api/src/backend/api/migrations/0001_initial.py index d2a8a39d3b..dc492f17c5 100644 --- a/api/src/backend/api/migrations/0001_initial.py +++ b/api/src/backend/api/migrations/0001_initial.py @@ -22,36 +22,36 @@ import api.rls from api.db_utils import ( - PostgresEnumMigration, - MemberRoleEnumField, + DB_PROWLER_PASSWORD, + DB_PROWLER_USER, + POSTGRES_TENANT_VAR, + POSTGRES_USER_VAR, + TASK_RUNNER_DB_TABLE, + InvitationStateEnum, + InvitationStateEnumField, MemberRoleEnum, + MemberRoleEnumField, + PostgresEnumMigration, ProviderEnum, ProviderEnumField, ProviderSecretTypeEnum, ProviderSecretTypeEnumField, ScanTriggerEnum, - StateEnumField, - StateEnum, ScanTriggerEnumField, - InvitationStateEnum, - InvitationStateEnumField, + StateEnum, + StateEnumField, register_enum, - DB_PROWLER_USER, - DB_PROWLER_PASSWORD, - TASK_RUNNER_DB_TABLE, - POSTGRES_TENANT_VAR, - POSTGRES_USER_VAR, ) from api.models import ( + Finding, + Invitation, + Membership, Provider, + ProviderSecret, Scan, + SeverityChoices, StateChoices, - Finding, StatusChoices, - SeverityChoices, - Membership, - ProviderSecret, - Invitation, ) DB_NAME = settings.DATABASES["default"]["NAME"] @@ -289,7 +289,8 @@ class Migration(migrations.Migration): ), ), # Enable tenants RLS based on memberships - migrations.RunSQL(f""" + migrations.RunSQL( + f""" ALTER TABLE tenants ENABLE ROW LEVEL SECURITY; -- Policy for SELECT @@ -364,7 +365,8 @@ class Migration(migrations.Migration): FOR INSERT TO {DB_PROWLER_USER} WITH CHECK (true); - """), + """ + ), # Create and register ProviderEnum type migrations.RunPython( ProviderEnumMigration.create_enum_type, @@ -1482,4 +1484,81 @@ class Migration(migrations.Migration): name="comp_ov_cp_id_req_fail_idx", ), ), + migrations.CreateModel( + name="ScanSummary", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("inserted_at", models.DateTimeField(auto_now_add=True)), + ("check_id", models.CharField(max_length=100)), + ("service", models.TextField()), + ( + "severity", + api.db_utils.SeverityEnumField( + choices=[ + ("critical", "Critical"), + ("high", "High"), + ("medium", "Medium"), + ("low", "Low"), + ("informational", "Informational"), + ] + ), + ), + ("region", models.TextField()), + ("_pass", models.IntegerField(db_column="pass", default=0)), + ("fail", models.IntegerField(default=0)), + ("muted", models.IntegerField(default=0)), + ("total", models.IntegerField(default=0)), + ("new", models.IntegerField(default=0)), + ("changed", models.IntegerField(default=0)), + ("unchanged", models.IntegerField(default=0)), + ("fail_new", models.IntegerField(default=0)), + ("fail_changed", models.IntegerField(default=0)), + ("pass_new", models.IntegerField(default=0)), + ("pass_changed", models.IntegerField(default=0)), + ("muted_new", models.IntegerField(default=0)), + ("muted_changed", models.IntegerField(default=0)), + ( + "scan", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="aggregations", + related_query_name="aggregation", + to="api.scan", + ), + ), + ( + "tenant", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="api.tenant" + ), + ), + ], + options={ + "db_table": "scan_summaries", + "abstract": False, + }, + ), + migrations.AddConstraint( + model_name="scansummary", + constraint=api.rls.RowLevelSecurityConstraint( + "tenant_id", + name="rls_on_scansummary", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ), + migrations.AddConstraint( + model_name="scansummary", + constraint=models.UniqueConstraint( + fields=("tenant", "scan", "check_id", "service", "severity", "region"), + name="unique_scan_summary", + ), + ), ] diff --git a/api/src/backend/api/models.py b/api/src/backend/api/models.py index f498988f88..664bc818fb 100644 --- a/api/src/backend/api/models.py +++ b/api/src/backend/api/models.py @@ -1,6 +1,6 @@ import json import re -from uuid import uuid4, UUID +from uuid import UUID, uuid4 from cryptography.fernet import Fernet from django.conf import settings @@ -11,35 +11,33 @@ from django.db import models from django.utils.translation import gettext_lazy as _ from django_celery_results.models import TaskResult -from prowler.lib.check.models import Severity from psqlextra.models import PostgresPartitionedModel from psqlextra.types import PostgresPartitioningMethod from uuid6 import uuid7 from api.db_utils import ( + CustomUserManager, + FindingDeltaEnumField, + InvitationStateEnumField, MemberRoleEnumField, - enum_to_choices, ProviderEnumField, - StateEnumField, + ProviderSecretTypeEnumField, ScanTriggerEnumField, - FindingDeltaEnumField, SeverityEnumField, + StateEnumField, StatusEnumField, - CustomUserManager, - ProviderSecretTypeEnumField, - InvitationStateEnumField, - one_week_from_now, + enum_to_choices, generate_random_token, + one_week_from_now, ) from api.exceptions import ModelValidationError from api.rls import ( + BaseSecurityConstraint, + RowLevelSecurityConstraint, RowLevelSecurityProtectedModel, -) -from api.rls import ( Tenant, - RowLevelSecurityConstraint, - BaseSecurityConstraint, ) +from prowler.lib.check.models import Severity fernet = Fernet(settings.SECRETS_ENCRYPTION_KEY.encode()) @@ -856,3 +854,51 @@ class Meta(RowLevelSecurityProtectedModel.Meta): class JSONAPIMeta: resource_name = "compliance-overviews" + + +class ScanSummary(RowLevelSecurityProtectedModel): + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) + inserted_at = models.DateTimeField(auto_now_add=True, editable=False) + check_id = models.CharField(max_length=100, blank=False, null=False) + service = models.TextField(blank=False) + severity = SeverityEnumField(choices=SeverityChoices) + region = models.TextField(blank=False) + _pass = models.IntegerField(db_column="pass", default=0) + fail = models.IntegerField(default=0) + muted = models.IntegerField(default=0) + total = models.IntegerField(default=0) + new = models.IntegerField(default=0) + changed = models.IntegerField(default=0) + unchanged = models.IntegerField(default=0) + + fail_new = models.IntegerField(default=0) + fail_changed = models.IntegerField(default=0) + pass_new = models.IntegerField(default=0) + pass_changed = models.IntegerField(default=0) + muted_new = models.IntegerField(default=0) + muted_changed = models.IntegerField(default=0) + + scan = models.ForeignKey( + Scan, + on_delete=models.CASCADE, + related_name="aggregations", + related_query_name="aggregation", + ) + + class Meta(RowLevelSecurityProtectedModel.Meta): + db_table = "scan_summaries" + + constraints = [ + models.UniqueConstraint( + fields=("tenant", "scan", "check_id", "service", "severity", "region"), + name="unique_scan_summary", + ), + RowLevelSecurityConstraint( + field="tenant_id", + name="rls_on_%(class)s", + statements=["SELECT", "INSERT", "UPDATE", "DELETE"], + ), + ] + + class JSONAPIMeta: + resource_name = "scan-summaries" diff --git a/api/src/backend/api/specs/v1.yaml b/api/src/backend/api/specs/v1.yaml index 755ab8c8df..fdc83418a3 100644 --- a/api/src/backend/api/specs/v1.yaml +++ b/api/src/backend/api/specs/v1.yaml @@ -780,16 +780,332 @@ paths: schema: $ref: '#/components/schemas/OpenApiResponseResponse' description: '' + /api/v1/overviews/findings: + get: + operationId: overviews_findings_retrieve + description: Fetch aggregated findings data across all providers, grouped by + various metrics such as passed, failed, muted, and total findings. This endpoint + calculates summary statistics based on the latest scans for each provider + and applies any provided filters, such as region, provider type, and scan + date. + summary: Get aggregated findings data + parameters: + - in: query + name: fields[findings-overview] + schema: + type: array + items: + type: string + enum: + - id + - new + - changed + - unchanged + - fail_new + - fail_changed + - pass_new + - pass_changed + - muted_new + - muted_changed + - total + - fail + - muted + - pass + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__date] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date-time + - in: query + name: filter[muted_findings] + schema: + type: boolean + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - new + - -new + - changed + - -changed + - unchanged + - -unchanged + - fail_new + - -fail_new + - fail_changed + - -fail_changed + - pass_new + - -pass_new + - pass_changed + - -pass_changed + - muted_new + - -muted_new + - muted_changed + - -muted_changed + - total + - -total + - fail + - -fail + - muted + - -muted + - pass + - -pass + explode: false + tags: + - Overview + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/OverviewFindingResponse' + description: '' + /api/v1/overviews/findings_severity: + get: + operationId: overviews_findings_severity_retrieve + description: Retrieve an aggregated summary of findings grouped by severity + levels, such as low, medium, high, and critical. The response includes the + total count of findings for each severity, considering only the latest scans + for each provider. Additional filters can be applied to narrow down results + by region, provider type, or other attributes. + summary: Get findings data by severity + parameters: + - in: query + name: fields[findings-severity-overview] + schema: + type: array + items: + type: string + enum: + - id + - critical + - high + - medium + - low + - informational + description: endpoint return only specific fields in the response on a per-type + basis by including a fields[TYPE] query parameter. + explode: false + - in: query + name: filter[inserted_at] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__date] + schema: + type: string + format: date + - in: query + name: filter[inserted_at__gte] + schema: + type: string + format: date-time + - in: query + name: filter[inserted_at__lte] + schema: + type: string + format: date-time + - in: query + name: filter[muted_findings] + schema: + type: boolean + - in: query + name: filter[provider_id] + schema: + type: string + format: uuid + - in: query + name: filter[provider_type] + schema: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + description: |- + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + - in: query + name: filter[provider_type__in] + schema: + type: array + items: + type: string + enum: + - aws + - azure + - gcp + - kubernetes + description: |- + Multiple values may be separated by commas. + + * `aws` - AWS + * `azure` - Azure + * `gcp` - GCP + * `kubernetes` - Kubernetes + explode: false + style: form + - in: query + name: filter[region] + schema: + type: string + - in: query + name: filter[region__icontains] + schema: + type: string + - in: query + name: filter[region__in] + schema: + type: array + items: + type: string + description: Multiple values may be separated by commas. + explode: false + style: form + - name: filter[search] + required: false + in: query + description: A search term. + schema: + type: string + - name: sort + required: false + in: query + description: '[list of fields to sort by](https://jsonapi.org/format/#fetching-sorting)' + schema: + type: array + items: + type: string + enum: + - id + - -id + - critical + - -critical + - high + - -high + - medium + - -medium + - low + - -low + - informational + - -informational + explode: false + tags: + - Overview + security: + - jwtAuth: [] + responses: + '200': + content: + application/vnd.api+json: + schema: + $ref: '#/components/schemas/OverviewSeverityResponse' + description: '' /api/v1/overviews/providers: get: operationId: overviews_providers_retrieve - description: Fetch aggregated summaries of the latest findings and resources - for each provider. This includes counts of passed, failed, and manual findings, - as well as the total number of resources managed by each provider. - summary: List aggregated overview data for providers + description: Retrieve an aggregated overview of findings and resources grouped + by providers. The response includes the count of passed, failed, and manual + findings, along with the total number of resources managed by each provider. + Only the latest findings for each provider are considered in the aggregation + to ensure accurate and up-to-date insights. + summary: Get aggregated provider data parameters: - in: query - name: fields[provider-overviews] + name: fields[providers-overview] schema: type: array items: @@ -4400,6 +4716,77 @@ components: $ref: '#/components/schemas/Membership' required: - data + OverviewFinding: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/OverviewFindingTypeEnum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: {} + attributes: + type: object + properties: + id: + type: string + default: n/a + new: + type: integer + changed: + type: integer + unchanged: + type: integer + fail_new: + type: integer + fail_changed: + type: integer + pass_new: + type: integer + pass_changed: + type: integer + muted_new: + type: integer + muted_changed: + type: integer + total: + type: integer + fail: + type: integer + muted: + type: integer + pass: + type: integer + required: + - new + - changed + - unchanged + - fail_new + - fail_changed + - pass_new + - pass_changed + - muted_new + - muted_changed + - total + - fail + - muted + - pass + OverviewFindingResponse: + type: object + properties: + data: + $ref: '#/components/schemas/OverviewFinding' + required: + - data + OverviewFindingTypeEnum: + type: string + enum: + - findings-overview OverviewProvider: type: object required: @@ -4449,7 +4836,54 @@ components: OverviewProviderTypeEnum: type: string enum: - - provider-overviews + - providers-overview + OverviewSeverity: + type: object + required: + - type + - id + additionalProperties: false + properties: + type: + allOf: + - $ref: '#/components/schemas/OverviewSeverityTypeEnum' + description: The [type](https://jsonapi.org/format/#document-resource-object-identification) + member is used to describe resource objects that share common attributes + and relationships. + id: {} + attributes: + type: object + properties: + id: + type: string + default: n/a + critical: + type: integer + high: + type: integer + medium: + type: integer + low: + type: integer + informational: + type: integer + required: + - critical + - high + - medium + - low + - informational + OverviewSeverityResponse: + type: object + properties: + data: + $ref: '#/components/schemas/OverviewSeverity' + required: + - data + OverviewSeverityTypeEnum: + type: string + enum: + - findings-severity-overview PaginatedComplianceOverviewList: type: object required: diff --git a/api/src/backend/api/tests/test_views.py b/api/src/backend/api/tests/test_views.py index 79589968bd..1178f6c276 100644 --- a/api/src/backend/api/tests/test_views.py +++ b/api/src/backend/api/tests/test_views.py @@ -1,29 +1,24 @@ import json -from datetime import datetime -from datetime import timezone, timedelta +from datetime import datetime, timedelta, timezone from unittest.mock import ANY, Mock, patch import jwt import pytest +from conftest import API_JSON_CONTENT_TYPE, TEST_PASSWORD, TEST_USER from django.urls import reverse from rest_framework import status from api.models import ( - User, + Invitation, Membership, Provider, ProviderGroup, ProviderGroupMembership, - Scan, ProviderSecret, - Invitation, + Scan, + User, ) from api.rls import Tenant -from conftest import ( - API_JSON_CONTENT_TYPE, - TEST_PASSWORD, - TEST_USER, -) TODAY = str(datetime.today().date()) @@ -1794,7 +1789,7 @@ def test_scans_invalid_retrieve(self, authenticated_client): ], ) @patch("api.v1.views.Task.objects.get") - @patch("api.v1.views.perform_scan_task.delay") + @patch("api.v1.views.perform_scan_task.apply_async") def test_scans_create_valid( self, mock_perform_scan_task, @@ -3009,9 +3004,9 @@ def test_invitations_filters( response = authenticated_client.get( reverse("invitation-list"), { - f"filter[{filter_name}]": filter_value - if filter_name != "inviter" - else str(user.id) + f"filter[{filter_name}]": ( + filter_value if filter_name != "inviter" else str(user.id) + ) }, ) @@ -3262,3 +3257,5 @@ def test_overview_providers_list( assert response.json()["data"][0]["attributes"]["resources"]["total"] == len( resources_fixture ) + + # TODO Add more tests for the rest of overviews diff --git a/api/src/backend/api/v1/serializers.py b/api/src/backend/api/v1/serializers.py index 9b0895adcb..9cc28bf009 100644 --- a/api/src/backend/api/v1/serializers.py +++ b/api/src/backend/api/v1/serializers.py @@ -1,5 +1,5 @@ import json -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone from django.conf import settings from django.contrib.auth import authenticate @@ -14,24 +14,23 @@ from rest_framework_simplejwt.tokens import RefreshToken from api.models import ( - StateChoices, - User, + ComplianceOverview, + Finding, + Invitation, Membership, Provider, ProviderGroup, ProviderGroupMembership, - Scan, - Task, + ProviderSecret, Resource, ResourceTag, - Finding, - ProviderSecret, - Invitation, - ComplianceOverview, + Scan, + StateChoices, + Task, + User, ) from api.rls import Tenant - # Tokens @@ -1234,7 +1233,7 @@ class OverviewProviderSerializer(serializers.Serializer): resources = serializers.SerializerMethodField(read_only=True) class JSONAPIMeta: - resource_name = "provider-overviews" + resource_name = "providers-overview" def get_root_meta(self, _resource, _many): return {"version": "v1"} @@ -1270,3 +1269,45 @@ def get_resources(self, obj): return { "total": obj["total_resources"], } + + +class OverviewFindingSerializer(serializers.Serializer): + id = serializers.CharField(default="n/a") + new = serializers.IntegerField() + changed = serializers.IntegerField() + unchanged = serializers.IntegerField() + fail_new = serializers.IntegerField() + fail_changed = serializers.IntegerField() + pass_new = serializers.IntegerField() + pass_changed = serializers.IntegerField() + muted_new = serializers.IntegerField() + muted_changed = serializers.IntegerField() + total = serializers.IntegerField() + _pass = serializers.IntegerField() + fail = serializers.IntegerField() + muted = serializers.IntegerField() + + class JSONAPIMeta: + resource_name = "findings-overview" + + def get_root_meta(self, _resource, _many): + return {"version": "v1"} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["pass"] = self.fields.pop("_pass") + + +class OverviewSeveritySerializer(serializers.Serializer): + id = serializers.CharField(default="n/a") + critical = serializers.IntegerField() + high = serializers.IntegerField() + medium = serializers.IntegerField() + low = serializers.IntegerField() + informational = serializers.IntegerField() + + class JSONAPIMeta: + resource_name = "findings-severity-overview" + + def get_root_meta(self, _resource, _many): + return {"version": "v1"} diff --git a/api/src/backend/api/v1/views.py b/api/src/backend/api/v1/views.py index 70af1069ab..90db12150a 100644 --- a/api/src/backend/api/v1/views.py +++ b/api/src/backend/api/v1/views.py @@ -2,20 +2,20 @@ from django.conf import settings as django_settings from django.contrib.postgres.search import SearchQuery from django.db import transaction -from django.db.models import Prefetch, Subquery, OuterRef, Count, Q, F +from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, Sum from django.urls import reverse from django.utils.decorators import method_decorator from django.views.decorators.cache import cache_control from drf_spectacular.settings import spectacular_settings from drf_spectacular.utils import ( - extend_schema, - extend_schema_view, OpenApiParameter, OpenApiResponse, OpenApiTypes, + extend_schema, + extend_schema_view, ) from drf_spectacular.views import SpectacularAPIView -from rest_framework import status, permissions +from rest_framework import permissions, status from rest_framework.decorators import action from rest_framework.exceptions import ( MethodNotAllowed, @@ -23,82 +23,87 @@ PermissionDenied, ValidationError, ) -from rest_framework.generics import get_object_or_404, GenericAPIView +from rest_framework.generics import GenericAPIView, get_object_or_404 from rest_framework_json_api.views import Response -from rest_framework_simplejwt.exceptions import InvalidToken -from rest_framework_simplejwt.exceptions import TokenError +from rest_framework_simplejwt.exceptions import InvalidToken, TokenError +from tasks.beat import schedule_provider_scan +from tasks.tasks import ( + check_provider_connection_task, + delete_provider_task, + perform_scan_summary_task, + perform_scan_task, +) -from api.base_views import BaseTenantViewset, BaseRLSViewSet, BaseUserViewset +from api.base_views import BaseRLSViewSet, BaseTenantViewset, BaseUserViewset from api.db_router import MainRouter from api.filters import ( + ComplianceOverviewFilter, + FindingFilter, + InvitationFilter, + MembershipFilter, ProviderFilter, ProviderGroupFilter, - TenantFilter, - MembershipFilter, + ProviderSecretFilter, + ResourceFilter, ScanFilter, + ScanSummaryFilter, TaskFilter, - ResourceFilter, - FindingFilter, - ProviderSecretFilter, - InvitationFilter, + TenantFilter, UserFilter, - ComplianceOverviewFilter, ) from api.models import ( - StatusChoices, - User, + ComplianceOverview, + Finding, + Invitation, Membership, Provider, ProviderGroup, ProviderGroupMembership, + ProviderSecret, + Resource, Scan, + ScanSummary, + SeverityChoices, + StatusChoices, Task, - Resource, - Finding, - ProviderSecret, - Invitation, - ComplianceOverview, + User, ) from api.pagination import ComplianceOverviewPagination from api.rls import Tenant from api.utils import validate_invitation from api.uuid_utils import datetime_to_uuid7 from api.v1.serializers import ( - TokenSerializer, - TokenRefreshSerializer, - UserSerializer, - UserCreateSerializer, - UserUpdateSerializer, + ComplianceOverviewFullSerializer, + ComplianceOverviewSerializer, + FindingSerializer, + InvitationAcceptSerializer, + InvitationCreateSerializer, + InvitationSerializer, + InvitationUpdateSerializer, MembershipSerializer, + OverviewFindingSerializer, + OverviewProviderSerializer, + OverviewSeveritySerializer, + ProviderCreateSerializer, + ProviderGroupMembershipUpdateSerializer, ProviderGroupSerializer, ProviderGroupUpdateSerializer, - ProviderGroupMembershipUpdateSerializer, + ProviderSecretCreateSerializer, + ProviderSecretSerializer, + ProviderSecretUpdateSerializer, ProviderSerializer, - ProviderCreateSerializer, ProviderUpdateSerializer, - TenantSerializer, - TaskSerializer, - ScanSerializer, + ResourceSerializer, ScanCreateSerializer, + ScanSerializer, ScanUpdateSerializer, - ResourceSerializer, - FindingSerializer, - ProviderSecretSerializer, - ProviderSecretUpdateSerializer, - ProviderSecretCreateSerializer, - InvitationSerializer, - InvitationCreateSerializer, - InvitationUpdateSerializer, - InvitationAcceptSerializer, - ComplianceOverviewSerializer, - ComplianceOverviewFullSerializer, - OverviewProviderSerializer, -) -from tasks.beat import schedule_provider_scan -from tasks.tasks import ( - check_provider_connection_task, - delete_provider_task, - perform_scan_task, + TaskSerializer, + TenantSerializer, + TokenRefreshSerializer, + TokenSerializer, + UserCreateSerializer, + UserSerializer, + UserUpdateSerializer, ) CACHE_DECORATOR = cache_control( @@ -803,12 +808,18 @@ def create(self, request, *args, **kwargs): with transaction.atomic(): scan = input_serializer.save() with transaction.atomic(): - task = perform_scan_task.delay( - tenant_id=request.tenant_id, - scan_id=str(scan.id), - provider_id=str(scan.provider_id), - # Disabled for now - # checks_to_execute=scan.scanner_args.get("checks_to_execute"), + task = perform_scan_task.apply_async( + kwargs={ + "tenant_id": request.tenant_id, + "scan_id": str(scan.id), + "provider_id": str(scan.provider_id), + # Disabled for now + # checks_to_execute=scan.scanner_args.get("checks_to_execute"), + }, + link=perform_scan_summary_task.si( + tenant_id=request.tenant_id, + scan_id=str(scan.id), + ), ) scan.task_id = task.id @@ -1295,26 +1306,67 @@ def list(self, request, *args, **kwargs): @extend_schema(tags=["Overview"]) @extend_schema_view( providers=extend_schema( - summary="List aggregated overview data for providers", - description="Fetch aggregated summaries of the latest findings and resources for each provider. " - "This includes counts of passed, failed, and manual findings, as well as the total number " - "of resources managed by each provider.", + summary="Get aggregated provider data", + description=( + "Retrieve an aggregated overview of findings and resources grouped by providers. " + "The response includes the count of passed, failed, and manual findings, along with " + "the total number of resources managed by each provider. Only the latest findings for " + "each provider are considered in the aggregation to ensure accurate and up-to-date insights." + ), + ), + findings=extend_schema( + summary="Get aggregated findings data", + description=( + "Fetch aggregated findings data across all providers, grouped by various metrics such as " + "passed, failed, muted, and total findings. This endpoint calculates summary statistics " + "based on the latest scans for each provider and applies any provided filters, such as " + "region, provider type, and scan date." + ), + filters=True, + ), + findings_severity=extend_schema( + summary="Get findings data by severity", + description=( + "Retrieve an aggregated summary of findings grouped by severity levels, such as low, medium, " + "high, and critical. The response includes the total count of findings for each severity, " + "considering only the latest scans for each provider. Additional filters can be applied to " + "narrow down results by region, provider type, or other attributes." + ), + filters=True, ), ) @method_decorator(CACHE_DECORATOR, name="list") class OverviewViewSet(BaseRLSViewSet): queryset = ComplianceOverview.objects.all() http_method_names = ["get"] - ordering = ["compliance_id"] + ordering = ["-id"] def get_queryset(self): - return Finding.objects.all() + if self.action == "providers": + return Finding.objects.all() + elif self.action == "findings": + return ScanSummary.objects.all() + elif self.action == "findings_severity": + return ScanSummary.objects.all() + else: + return super().get_queryset() def get_serializer_class(self): if self.action == "providers": return OverviewProviderSerializer + elif self.action == "findings": + return OverviewFindingSerializer + elif self.action == "findings_severity": + return OverviewSeveritySerializer return super().get_serializer_class() + def get_filterset_class(self): + if self.action == "providers": + return None + elif self.action in ["findings", "findings_severity"]: + return ScanSummaryFilter + return None + @extend_schema(exclude=True) def list(self, request, *args, **kwargs): raise MethodNotAllowed(method="GET") @@ -1366,7 +1418,7 @@ def providers(self, request): for res in resources_aggregated if res["provider__provider"] == provider ), - 0, # Default to 0 if no resources are found + 0, ) overview.append( { @@ -1382,3 +1434,74 @@ def providers(self, request): serializer = OverviewProviderSerializer(overview, many=True) return Response(serializer.data, status=status.HTTP_200_OK) + + @action(detail=False, methods=["get"], url_name="findings") + def findings(self, request): + queryset = self.get_queryset() + filtered_queryset = self.filter_queryset(queryset) + + latest_scan_subquery = ( + Scan.objects.filter(provider_id=OuterRef("scan__provider_id")) + .order_by("-id") + .values("id")[:1] + ) + + annotated_queryset = filtered_queryset.annotate( + latest_scan_id=Subquery(latest_scan_subquery) + ) + + filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id")) + + aggregated_totals = filtered_queryset.aggregate( + _pass=Sum("_pass") or 0, + fail=Sum("fail") or 0, + muted=Sum("muted") or 0, + total=Sum("total") or 0, + new=Sum("new") or 0, + changed=Sum("changed") or 0, + unchanged=Sum("unchanged") or 0, + fail_new=Sum("fail_new") or 0, + fail_changed=Sum("fail_changed") or 0, + pass_new=Sum("pass_new") or 0, + pass_changed=Sum("pass_changed") or 0, + muted_new=Sum("muted_new") or 0, + muted_changed=Sum("muted_changed") or 0, + ) + + for key in aggregated_totals: + if aggregated_totals[key] is None: + aggregated_totals[key] = 0 + + serializer = self.get_serializer(aggregated_totals) + return Response(serializer.data, status=status.HTTP_200_OK) + + @action(detail=False, methods=["get"], url_name="findings_severity") + def findings_severity(self, request): + queryset = self.get_queryset() + filtered_queryset = self.filter_queryset(queryset) + + latest_scan_subquery = ( + Scan.objects.filter(provider_id=OuterRef("scan__provider_id")) + .order_by("-id") + .values("id")[:1] + ) + + annotated_queryset = filtered_queryset.annotate( + latest_scan_id=Subquery(latest_scan_subquery) + ) + + filtered_queryset = annotated_queryset.filter(scan_id=F("latest_scan_id")) + + severity_counts = ( + filtered_queryset.values("severity") + .annotate(count=Sum("total")) + .order_by("severity") + ) + + severity_data = {sev[0]: 0 for sev in SeverityChoices} + + for item in severity_counts: + severity_data[item["severity"]] = item["count"] + + serializer = OverviewSeveritySerializer(severity_data) + return Response(serializer.data, status=status.HTTP_200_OK) diff --git a/api/src/backend/tasks/jobs/scan.py b/api/src/backend/tasks/jobs/scan.py index 2b2fea8f17..5e84797377 100644 --- a/api/src/backend/tasks/jobs/scan.py +++ b/api/src/backend/tasks/jobs/scan.py @@ -3,8 +3,7 @@ from datetime import datetime, timezone from celery.utils.log import get_task_logger -from prowler.lib.outputs.finding import Finding as ProwlerFinding -from prowler.lib.scan.scan import Scan as ProwlerScan +from django.db.models import Case, Count, IntegerField, Sum, When from api.compliance import ( PROWLER_COMPLIANCE_OVERVIEW_TEMPLATE, @@ -12,17 +11,20 @@ ) from api.db_utils import tenant_transaction from api.models import ( - Provider, - Scan, + ComplianceOverview, Finding, + Provider, Resource, ResourceTag, - StatusChoices as FindingStatus, + Scan, + ScanSummary, StateChoices, - ComplianceOverview, ) +from api.models import StatusChoices as FindingStatus from api.utils import initialize_prowler_provider from api.v1.serializers import ScanTaskSerializer +from prowler.lib.outputs.finding import Finding as ProwlerFinding +from prowler.lib.scan.scan import Scan as ProwlerScan logger = get_task_logger(__name__) @@ -268,7 +270,7 @@ def perform_prowler_scan( scan_instance.unique_resource_count = len(unique_resources) scan_instance.save() - if generate_compliance: + if exception is None and generate_compliance: try: regions = prowler_provider.get_regions() except AttributeError: @@ -321,3 +323,155 @@ def perform_prowler_scan( serializer = ScanTaskSerializer(instance=scan_instance) return serializer.data + + +def aggregate_findings(tenant_id: str, scan_id: str): + """ + Aggregates findings for a given scan and stores the results in the ScanSummary table. + + This function retrieves all findings associated with a given `scan_id` and calculates various + metrics such as counts of failed, passed, and muted findings, as well as their deltas (new, + changed, unchanged). The results are grouped by `check_id`, `service`, `severity`, and `region`. + These aggregated metrics are then stored in the `ScanSummary` table. + + Args: + tenant_id (str): The ID of the tenant to which the scan belongs. + scan_id (str): The ID of the scan for which findings need to be aggregated. + + Aggregated Metrics: + - fail: Total number of failed findings. + - _pass: Total number of passed findings. + - muted: Total number of muted findings. + - total: Total number of findings. + - new: Total number of new findings. + - changed: Total number of changed findings. + - unchanged: Total number of unchanged findings. + - fail_new: Failed findings with a delta of 'new'. + - fail_changed: Failed findings with a delta of 'changed'. + - pass_new: Passed findings with a delta of 'new'. + - pass_changed: Passed findings with a delta of 'changed'. + - muted_new: Muted findings with a delta of 'new'. + - muted_changed: Muted findings with a delta of 'changed'. + """ + with tenant_transaction(tenant_id): + findings = Finding.objects.filter(scan_id=scan_id) + + aggregation = findings.values( + "check_id", + "resources__service", + "severity", + "resources__region", + ).annotate( + fail=Sum( + Case( + When(status="FAIL", then=1), + default=0, + output_field=IntegerField(), + ) + ), + _pass=Sum( + Case( + When(status="PASS", then=1), + default=0, + output_field=IntegerField(), + ) + ), + muted=Sum( + Case( + When(status="MUTED", then=1), + default=0, + output_field=IntegerField(), + ) + ), + total=Count("id"), + new=Sum( + Case( + When(delta="new", then=1), + default=0, + output_field=IntegerField(), + ) + ), + changed=Sum( + Case( + When(delta="changed", then=1), + default=0, + output_field=IntegerField(), + ) + ), + unchanged=Sum( + Case( + When(delta__isnull=True, then=1), + default=0, + output_field=IntegerField(), + ) + ), + fail_new=Sum( + Case( + When(delta="new", status="FAIL", then=1), + default=0, + output_field=IntegerField(), + ) + ), + fail_changed=Sum( + Case( + When(delta="changed", status="FAIL", then=1), + default=0, + output_field=IntegerField(), + ) + ), + pass_new=Sum( + Case( + When(delta="new", status="PASS", then=1), + default=0, + output_field=IntegerField(), + ) + ), + pass_changed=Sum( + Case( + When(delta="changed", status="PASS", then=1), + default=0, + output_field=IntegerField(), + ) + ), + muted_new=Sum( + Case( + When(delta="new", status="MUTED", then=1), + default=0, + output_field=IntegerField(), + ) + ), + muted_changed=Sum( + Case( + When(delta="changed", status="MUTED", then=1), + default=0, + output_field=IntegerField(), + ) + ), + ) + + with tenant_transaction(tenant_id): + scan_aggregations = { + ScanSummary( + tenant_id=tenant_id, + scan_id=scan_id, + check_id=agg["check_id"], + service=agg["resources__service"], + severity=agg["severity"], + region=agg["resources__region"], + fail=agg["fail"], + _pass=agg["_pass"], + muted=agg["muted"], + total=agg["total"], + new=agg["new"], + changed=agg["changed"], + unchanged=agg["unchanged"], + fail_new=agg["fail_new"], + fail_changed=agg["fail_changed"], + pass_new=agg["pass_new"], + pass_changed=agg["pass_changed"], + muted_new=agg["muted_new"], + muted_changed=agg["muted_changed"], + ) + for agg in aggregation + } + ScanSummary.objects.bulk_create(scan_aggregations, batch_size=3000) diff --git a/api/src/backend/tasks/tasks.py b/api/src/backend/tasks/tasks.py index 4fc933127e..1237b543c5 100644 --- a/api/src/backend/tasks/tasks.py +++ b/api/src/backend/tasks/tasks.py @@ -1,12 +1,12 @@ from celery import shared_task +from config.celery import RLSTask +from tasks.jobs.connection import check_provider_connection +from tasks.jobs.deletion import delete_instance +from tasks.jobs.scan import aggregate_findings, perform_prowler_scan from api.db_utils import tenant_transaction from api.decorators import set_tenant from api.models import Provider, Scan -from config.celery import RLSTask -from tasks.jobs.connection import check_provider_connection -from tasks.jobs.deletion import delete_instance -from tasks.jobs.scan import perform_prowler_scan @shared_task(base=RLSTask, name="provider-connection-check") @@ -110,3 +110,8 @@ def perform_scheduled_scan_task(self, tenant_id: str, provider_id: str): scan_id=str(scan_instance.id), provider_id=provider_id, ) + + +@shared_task(name="scan-summary") +def perform_scan_summary_task(tenant_id: str, scan_id: str): + return aggregate_findings(tenant_id=tenant_id, scan_id=scan_id) diff --git a/api/src/backend/tasks/tests/test_scan.py b/api/src/backend/tasks/tests/test_scan.py index 798ebcb354..da79f78555 100644 --- a/api/src/backend/tasks/tests/test_scan.py +++ b/api/src/backend/tasks/tests/test_scan.py @@ -1,19 +1,19 @@ -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch import pytest +from tasks.jobs.scan import ( + _create_finding_delta, + _store_resources, + perform_prowler_scan, +) from api.models import ( - StateChoices, - Severity, Finding, + Provider, Resource, + Severity, + StateChoices, StatusChoices, - Provider, -) -from tasks.jobs.scan import ( - perform_prowler_scan, - _create_finding_delta, - _store_resources, ) @@ -358,3 +358,6 @@ def test_store_resources_with_tags( assert resource == resource_instance assert resource_uid_tuple == (resource_instance.uid, resource_instance.region) + + +# TODO Add tests for aggregations