Add WAS scan and help text to models

Add WAS scan updates and help text to models

src/pe_reports/pe_reports_django_project/dataAPI/schemas.py

@@ -4100,3 +4100,156 @@ class OrgAssetTaskResp(BaseModel):
     status: str
     result: Optional[OrgAssetPagedResult] = None
     error: Optional[str] = None
+
+
+class WasFindingInsert(BaseModel):
+    """WasFindingResult schema class"""
+    finding_uid: str
+    finding_type: Optional[str] = None
+    webapp_id: Optional[str] = None
+    was_org_id: Optional[str] = None
+    owasp_category: Optional[str] = None
+    severity: Optional[str] = None
+    times_detected: Optional[int] = None
+    base_score: Optional[float] = None
+    temporal_score: Optional[float] = None
+    fstatus: Optional[str] = None
+    last_detected: Optional[str] = None
+    first_detected: Optional[str] = None
+    is_remediated: Optional[bool] = None
+    potential: Optional[bool] = None
+    webapp_url: Optional[str] = None
+    webapp_name: Optional[str] = None
+    name: Optional[str] = None
+    cvss_v3_attack_vector: Optional[str] = None
+    cwe_list: Optional[List[str]] = None
+    wasc_list: Optional[List[Dict]] = None
+    last_tested: Optional[str] = None
+    fixed_date: Optional[str] = None
+    is_ignored: Optional[bool] = None
+    url: Optional[str] = None
+    qid: Optional[int] = None
+    response: Optional[str] = None
+
+class WasFindingResult(BaseModel):
+    """WasFindingResult schema class"""
+    finding_uid: str
+    finding_type: Optional[str] = None
+    webapp_id: Optional[str] = None
+    was_org_id: Optional[str] = None
+    owasp_category: Optional[str] = None
+    severity: Optional[str] = None
+    times_detected: Optional[int] = None
+    base_score: Optional[float] = None
+    temporal_score: Optional[float] = None
+    fstatus: Optional[str] = None
+    last_detected: Optional[str] = None
+    first_detected: Optional[str] = None
+    is_remediated: Optional[bool] = None
+    potential: Optional[bool] = None
+    webapp_url: Optional[str] = None
+    webapp_name: Optional[str] = None
+    name: Optional[str] = None
+    cvss_v3_attack_vector: Optional[str] = None
+    cwe_list: Optional[List[str]] = None
+    wasc_list: Optional[List[Dict]] = None
+    last_tested: Optional[str] = None
+    fixed_date: Optional[str] = None
+    is_ignored: Optional[bool] = None
+    url: Optional[str] = None
+    qid: Optional[int] = None
+    response: Optional[str] = None
+
+class WasFindingPagedResp(BaseModel):
+    """OrgAssetPagedResult schema class."""
+
+    total_pages: int
+    current_page: int
+    data: Optional[List[WasFindingResult]] = None
+
+class WasFindingsTaskResp(BaseModel):
+    """WasFindingsTaskResp schema class."""
+    task_id: str
+    status: str
+    result: Optional[WasFindingPagedResp] = None
+    error: Optional[str] = None
+
+
+class WasFindingsPagedInput(BaseModel):
+    """WasFindingsPagedInput schema class."""
+
+    org_acronym: str
+    page: int
+    per_page: int
+
+    class Config:
+        """WasFindingsPagedInput schema config class."""
+
+        orm_mode = True
+
+
+
+class WasReportInsert(BaseModel):
+    """WasReportInsert schema class."""
+
+    org_name: Optional[str] = Field(default=None)
+    date_pulled: Optional[str] = Field(default=None)
+    last_scan_date: Optional[str] = Field(default=None)
+    security_risk: Optional[str] = Field(default=None)
+    total_info: Optional[int] = Field(default=None)
+    num_apps: Optional[int] = Field(default=None)
+    risk_color: Optional[str] = Field(default=None)
+    sensitive_count: Optional[int] = Field(default=None)
+    sensitive_color: Optional[str] = Field(default=None)
+    max_days_open_urgent: Optional[int] = Field(default=None)
+    max_days_open_critical: Optional[int] = Field(default=None)
+    urgent_color: Optional[str] = Field(default=None)
+    critical_color: Optional[str] = Field(default=None)
+    org_was_acronym: Optional[str] = Field(default=None)
+    name_len: Optional[str] = Field(default=None)
+    vuln_csv_dict: Optional[Dict] = Field(default={})
+    ssn_cc_dict: Optional[Dict] = Field(default={})
+    app_overview_csv_dict: Optional[Dict] = Field(default={})
+    details_csv: Optional[List] = Field(default=[])
+    info_csv: Optional[List] = Field(default=[])
+    links_crawled: Optional[List] = Field(default=[])
+    links_rejected: Optional[List] = Field(default=[])
+    emails_found: Optional[List] = Field(default=[])
+    owasp_count_dict: Optional[Dict] = Field(default={})
+    group_count_dict: Optional[Dict] = Field(default={})
+    fixed: Optional[int] = Field(default=None)
+    total: Optional[int] = Field(default=None)
+    vulns_monthly_dict: Optional[Dict] = Field(default={})
+    path_disc: Optional[int] = Field(default=None)
+    info_disc: Optional[int] = Field(default=None)
+    cross_site: Optional[int] = Field(default=None)
+    burp: Optional[int] = Field(default=None)
+    sql_inj: Optional[int] = Field(default=None)
+    bugcrowd: Optional[int] = Field(default=None)
+    reopened: Optional[int] = Field(default=None)
+    reopened_color: Optional[str] = Field(default=None)
+    new_vulns: Optional[int] = Field(default=None)
+    new_vulns_color: Optional[str] = Field(default=None)
+    tot_vulns: Optional[int] = Field(default=None)
+    tot_vulns_color: Optional[str] = Field(default=None)
+    lev1: Optional[int] = Field(default=None)
+    lev2: Optional[int] = Field(default=None)
+    lev3: Optional[int] = Field(default=None)
+    lev4: Optional[int] = Field(default=None)
+    lev5: Optional[int] = Field(default=None)
+    severities: Optional[List[int]] = Field(default=[])
+    ages: Optional[List[int]] = Field(default=[])
+    pdf_obj: Optional[str] = Field(default=None)
+
+    class Config:
+        """WasReportInsert schema config class."""
+
+        orm_mode = True
+
+
+class WasReportTaskResp(BaseModel):
+    """WasReportTaskResp schema class."""
+    task_id: str
+    status: str
+    result: Optional[WasReportInsert] = None
+    error: Optional[str] = None

src/pe_reports/pe_reports_django_project/dataAPI/tasks.py

@@ -60,8 +60,16 @@
     VwShodanvulnsSuspected,
     VwShodanvulnsVerified,
     WasFindings,
+    WasReport,
     XpanseAlerts,
 )
+from dmz_mini_dl.models import (
+    CredentialBreaches as MDL_CredentialBreaches,
+    CredentialExposures as MDL_CredentialExposures,
+    DataSource as MDL_DataSource,
+    Organization as MDL_Organization,
+    WasReport as MDL_WasReport
+)
 
 # cisagov Libraries
 from pe_reports.helpers import ip_passthrough
@@ -1223,8 +1231,31 @@ def cred_breach_sixgill_task(self, new_breaches: List[dict]):
     """Task function for the cred_breaches_sixgill_insert API endpoint."""
     create_ct = 0
     update_ct = 0
+    try:
+        mdl_source_inst = MDL_DataSource.objects.get(name="Sixgill")
+    except MDL_DataSource.DoesNotExist:
+        LOGGER.warning(f"DataSource Sixgill not found.")
+        mdl_source_inst = None  # Set to None if DataSource is not found
     for new_breach in new_breaches:
         # Insert each row of data
+        try:
+            MDL_CredentialBreaches.objects.get(breach_name=new_breach["breach_name"])
+
+            MDL_CredentialBreaches.objects.filter(breach_name=new_breach["breach_name"]
+            ).update(
+                password_included=new_breach["password_included"],
+            )
+        except MDL_CredentialBreaches.DoesNotExist:
+            MDL_CredentialBreaches.objects.create(
+                credential_breaches_uid=uuid.uuid1(),
+                breach_name=new_breach["breach_name"],
+                description=new_breach["description"],
+                breach_date=new_breach["breach_date"],
+                password_included=new_breach["password_included"],
+                data_source=mdl_source_inst,
+                modified_date=new_breach["modified_date"],
+            )
+
         try:
             CredentialBreaches.objects.get(breach_name=new_breach["breach_name"])
             # If record already exists, update
@@ -1264,17 +1295,51 @@ def cred_exp_sixgill_task(self, new_exposures: List[dict]):
     """Task function for the credexp_insert API endpoint."""
     update_ct = 0
     create_ct = 0
+    try:
+        mdl_source_inst = MDL_DataSource.objects.get(name="Sixgill")
+    except MDL_DataSource.DoesNotExist:
+        LOGGER.warning(f"DataSource Sixgill not found.")
+        mdl_source_inst = None  # Set to None if DataSource is not found
     for new_exposure in new_exposures:
+        curr_org_inst = Organizations.objects.get(
+                organizations_uid=new_exposure["organizations_uid"]
+            )
+        try:
+            MDL_CredentialExposures.objects.get(
+                breach_name=new_exposure["breach_name"],
+                email=new_exposure["email"],
+            )
+        except MDL_CredentialExposures.DoesNotExist:
+            mdl_org_inst = MDL_Organization.objects.get(
+                        acronym=curr_org_inst.cyhy_db_name
+                    )
+
+            mdl_breach_inst = CredentialBreaches.objects.get(
+                breach_name=new_exposure["breach_name"]
+            )
+            CredentialExposures.objects.create(
+                credential_exposures_uid=uuid.uuid1(),
+                modified_date=new_exposure["modified_date"],
+                sub_domain=new_exposure["sub_domain"],
+                email=new_exposure["email"],
+                hash_type=new_exposure["hash_type"],
+                name=new_exposure["name"],
+                login_id=new_exposure["login_id"],
+                password=new_exposure["password"],
+                phone=new_exposure["phone"],
+                breach_name=new_exposure["breach_name"],
+                organization=mdl_org_inst,
+                data_source=mdl_source_inst,
+                credential_breaches=mdl_breach_inst,
+            )
         try:
             CredentialExposures.objects.get(
                 breach_name=new_exposure["breach_name"],
                 email=new_exposure["email"],
             )
         except CredentialExposures.DoesNotExist:
             # If cred exp record doesn't exist yet, create one
-            curr_org_inst = Organizations.objects.get(
-                organizations_uid=new_exposure["organizations_uid"]
-            )
+
             curr_source_inst = DataSource.objects.get(
                 data_source_uid=new_exposure["data_source_uid"]
             )
@@ -1658,4 +1723,96 @@ def get_orgs_and_assets(self, page: int, per_page: int):
         "current_page": page,
         "data": orgs_list,
     }
-    return result
+    return result
+
+def base64_to_bytes(base64_str) -> bytes:
+    """Convert a Base64-encoded string to binary data."""
+    if base64_str:
+        return base64.b64decode(base64_str)
+    else:
+        return base64_str
+
+@shared_task(bind=True)
+def insert_was_report(self, data):
+    # try:
+    LOGGER.info('Top of Was Report Task')
+    LOGGER.info(data)
+    defaults_dict={
+        "org_name": data.get('org_name'),
+        "date_pulled": data.get('date_pulled'),
+        "last_scan_date": data.get('last_scan_date'),
+        "security_risk": data.get('security_risk'),
+        "total_info": data.get('total_info'),
+        "num_apps": data.get('num_apps'),
+        "risk_color": data.get('risk_color'),
+        "sensitive_count": data.get('sensitive_count'),
+        "sensitive_color": data.get('sensitive_color'),
+        "max_days_open_urgent": data.get('max_days_open_urgent'),
+        "max_days_open_critical": data.get('max_days_open_critical'),
+        "urgent_color": data.get('urgent_color'),
+        "critical_color": data.get('critical_color'),
+        "name_len": data.get('name_len'),
+        "vuln_csv_dict": data.get('vuln_csv_dict'),
+        "ssn_cc_dict": data.get('ssn_cc_dict'),
+        "app_overview_csv_dict": data.get('app_overview_csv_dict'),
+        "details_csv": data.get('details_csv'),
+        "info_csv": data.get('info_csv'),
+        "links_crawled": data.get('links_crawled'),
+        "links_rejected": data.get('links_rejected'),
+        "emails_found": data.get('emails_found'),
+        "owasp_count_dict": data.get('owasp_count_dict'),
+        "group_count_dict": data.get('group_count_dict'),
+        "fixed": data.get('fixed'),
+        "total": data.get('total'),
+        "vulns_monthly_dict": data.get('vulns_monthly_dict'),
+        "path_disc": data.get('path_disc'),
+        "info_disc": data.get('info_disc'),
+        "cross_site": data.get('cross_site'),
+        "burp": data.get('burp'),
+        "sql_inj": data.get('sql_inj'),
+        "bugcrowd": data.get('bugcrowd'),
+        "reopened": data.get('reopened'),
+        "reopened_color": data.get('reopened_color'),
+        "new_vulns": data.get('new_vulns'),
+        "new_vulns_color": data.get('new_vulns_color'),
+        "tot_vulns": data.get('tot_vulns'),
+        "tot_vulns_color":  data.get('tot_vulns_color'),
+        "lev1": data.get('lev1'),
+        "lev2": data.get('lev2'),
+        "lev3": data.get('lev3'),
+        "lev4": data.get('lev4'),
+        "lev5": data.get('lev5'),
+        "severities": data.get('severities'),
+        "ages": data.get('ages'),
+        "pdf_obj": base64_to_bytes(data.get('pdf_obj'))
+
+    }
+    was_report_object, created = WasReport.objects.update_or_create(
+        org_was_acronym = data.get('org_was_acronym'),
+        last_scan_date = data.get('last_scan_date'),
+        defaults=defaults_dict
+    )
+    try:
+        mdl_was_report_object, mdl_created = MDL_WasReport.objects.update_or_create(
+        org_was_acronym = data.get('org_was_acronym'),
+        last_scan_date = data.get('last_scan_date'),
+        defaults=defaults_dict
+    )
+    except Exception:
+        LOGGER.info(f"Failed to insert WAS report for {data.get('org_was_acronym')}")
+
+
+    if created:
+        LOGGER.info("New Was record created for %s", data.get('org_was_acronym'))
+        return {
+            "message": "New Was record created.",
+            "was_report_id": was_report_object.id,
+        }
+    else:
+        return {"message": "Record updated successfully.", "was_report_id": was_report_object.id}
+
+    # except Exception as e:
+    #     LOGGER.error(e)
+    #     print("failed to insert or update")
+    #     return {"message": "Failed to insert or update.", "was_report_obj": None, "error": e}
+    #     LOGGER.info("API key expired please try again")