diff --git a/.gitignore b/.gitignore index 59400e1b..9176017a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ src/pe_reports/assets/ __pycache__ .coverage .mypy_cache +venv/ .pytest_cache .python-version *.egg-info @@ -53,4 +54,4 @@ dnstwist_output.txt adhoc_investigations/adhoc_investigation.ini adhoc_investigations/input_data adhoc_investigations/output_data -adhoc_investigations/dnsmonitor_monitored_domains.csv +adhoc_investigations/dnsmonitor_monitored_domains.csv \ No newline at end of file diff --git a/setup.py b/setup.py index 997067cc..94bfff47 100644 --- a/setup.py +++ b/setup.py @@ -129,8 +129,10 @@ def get_version(version_file): "matplotlib == 3.3.4", "nested-lookup", "openpyxl", - "pandas == 1.1.5", + "pandas", "pdfkit", + "presidio-analyzer", + "presidio-anonymizer", "psutil", "psycopg2-binary", "psycopg2-binary", @@ -147,12 +149,17 @@ def get_version(version_file): "reportlab", "requests", "schema == 0.7.5", + "scrubadub", "setuptools == 58.1.0", "scikit-learn", "shodan == 1.27.0", "sshtunnel", "sslyze>=5.0.0", - # "spacy", + "spacy", + "spacy-loggers", + "spacy-legacy", + "spacy-transformers", + "spacy-alignments", "nltk", "beautifulsoup4", "sublist3r", diff --git a/src/pe_reports/Dockerfile b/src/pe_reports/Dockerfile new file mode 100644 index 00000000..0674dfc3 --- /dev/null +++ b/src/pe_reports/Dockerfile @@ -0,0 +1,47 @@ +# Use the Python 3.10.2 image as the base image + +FROM python:3.10.2 + +# Install required tools +#RUN apt-get update && apt-get install -y curl build-essential + +# Install Rust and Cargo +#RUN curl https://sh.rustup.rs -sSf | sh -s -- -y + +# Add Rust to PATH +#ENV PATH="/root/.cargo/bin:${PATH}" + +# Install required tools +RUN apt-get update && apt-get install -y bash g++ gcc make redis redis-tools + +# Create non-root user +RUN useradd -m -u 1001 atc_api + +# Upgrade pip and certifi +RUN python3 -m pip install --upgrade pip && pip install --upgrade certifi + + + +# Set environment variables +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# Create working directory with correct ownership +RUN mkdir /code && chown atc_api:atc_api /code +WORKDIR /code + +# Install dependencies +COPY --chown=atc_api:atc_api ./pe_reports_django_project/pe_reports_django/requirements.txt /code/ +RUN pip install --no-cache-dir -r requirements.txt +# Copy the project code +COPY --chown=atc_api:atc_api pe_reports_django_project /code + +# Switch to non-root user +USER atc_api + +# Set Django environment variable +ENV DJANGO_SETTINGS_MODULE=pe_reports_django.settings + +# Run the application +CMD uvicorn --workers 4 pe_reports_django.asgi:app1 --host 0.0.0.0 --port 8000 --reload + diff --git a/src/pe_reports/__init__.py b/src/pe_reports/__init__.py index c9fbe7ac..648d7c11 100644 --- a/src/pe_reports/__init__.py +++ b/src/pe_reports/__init__.py @@ -1,105 +1,3 @@ """The pe_reports library.""" -# We disable a Flake8 check for "Module imported but unused (F401)" here because -# although this import is not directly used, it populates the value -# package_name.__version__, which is used to get version information about this -# Python package. - -# Standard Python Libraries -import logging -from logging.handlers import RotatingFileHandler -import os - -# Third-Party Libraries -# from celery import Celery -from flask import Flask, render_template -from flask_login import LoginManager -from flask_migrate import Migrate -from flask_sqlalchemy import SQLAlchemy - -# cisagov Libraries -from pe_reports.data.config import config - -from ._version import __version__ # noqa: F401 - -# Stakeholder views -# from pe_reports.home.views import home_blueprint -# from pe_reports.report_gen.views import report_gen_blueprint -# from pe_reports.stakeholder.views import stakeholder_blueprint -# from pe_reports.stakeholder_bulk_upload.views import stakeholder_bulk_upload_blueprint -# from pe_reports.stakeholder_full.views import stakeholder_full_blueprint - - -params = config() -login_manager = LoginManager() -# Flask implementation -app = Flask(__name__) -app.config["SECRET_KEY"] = os.getenv("FLASK_SECRET_KEY", "dev") -app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False -app.config[ - "SQLALCHEMY_DATABASE_URI" -] = f'postgresql+psycopg2://{params["user"]}:{params["password"]}@{params["host"]}:{params["port"]}/{params["database"]}' - - -# Configure the redis server -# app.config["CELERY_BROKER_URL"] = "redis://localhost:6379/0" -# app.config["CELERY_RESULT_BACKEND"] = "redis://localhost:6379/0" -app.config["UPLOAD_FOLDER"] = "src/pe_reports/uploads/" -app.config["ALLOWED_EXTENSIONS"] = {"txt", "csv"} CENTRAL_LOGGING_FILE = "pe_reports_logging.log" -DEBUG = False -# Setup Logging -"""Set up logging and call the run_pe_script function.""" -if DEBUG is True: - level = "DEBUG" -else: - level = "INFO" - -# Logging will rotate at 2GB -logging.basicConfig( - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - datefmt="%m/%d/%Y %I:%M:%S", - level=level, - handlers=[ - RotatingFileHandler(CENTRAL_LOGGING_FILE, maxBytes=2000000, backupCount=10) - ], -) - -app.config["LOGGER"] = logging.getLogger(__name__) - -# with open('username.txt', 'w') as file: -# file.write(pwd.getpwuid(os.getuid())[0]) - -# Creates a Celery object -# celery = Celery(app.name, broker=app.config["CELERY_BROKER_URL"]) -# celery.conf.update(app.config) - -# Config DB -db = SQLAlchemy(app) -Migrate(app, db) - -# TODO: Add a login page in the future. Issue #207 contains details -# login_manager.init_app(app) -# login_manager.login_view = "login" - -__all__ = ["app", "pages", "report_generator", "stylesheet"] - - -# Register the flask apps -# app.register_blueprint(stakeholder_blueprint) -# app.register_blueprint(stakeholder_full_blueprint) -# app.register_blueprint(stakeholder_bulk_upload_blueprint) -# app.register_blueprint(report_gen_blueprint) -# TODO: Add login blueprint. Issue #207 contains details -# app.register_blueprint(manage_login_blueprint) -# app.register_blueprint(home_blueprint) - - -@app.errorhandler(404) -def page_not_found(e): - return render_template("404.html") - - -if __name__ == "__main__": - logging.info("The program has started...") - app.run(host="127.0.0.1", debug=DEBUG, port=8000) diff --git a/src/pe_reports/data/db_query.py b/src/pe_reports/data/db_query.py index 7ebb0474..9bb8f34f 100644 --- a/src/pe_reports/data/db_query.py +++ b/src/pe_reports/data/db_query.py @@ -62,7 +62,7 @@ def task_api_call(task_url, check_url, data={}, retry_time=3): # Ping task status endpoint and get status # check_task_resp = requests.get(check_task_url, headers=headers).json() check_task_resp = requests.get(check_task_url, headers=headers) - #print(check_task_resp) + # print(check_task_resp) check_task_resp = check_task_resp.json() task_status = check_task_resp.get("status") LOGGER.info( @@ -2200,7 +2200,7 @@ def query_previous_period(org_uid, prev_end_date): return assets_dict -# ---------- PE-Score API Queries, Issue 635 ---------- +# ---------- PE-Score API Queries, Issue 635 ---------- # --- Issue 635 --- def pescore_hist_domain_alert(start_date, end_date): """ @@ -3609,7 +3609,7 @@ def upsert_new_cves_tsql(new_cves): # --- 018 atc-framework OLD TSQL --- -def get_demo_orgs(conn): +def get_demo_orgs_tsql(conn): """Query organizations table for orgs we report on.""" try: cur = conn.cursor() @@ -3623,4 +3623,4 @@ def get_demo_orgs(conn): LOGGER.error("There was a problem with your database query %s", error) finally: if conn is not None: - close(conn) \ No newline at end of file + close(conn) diff --git a/src/pe_reports/docker-compose.yaml b/src/pe_reports/docker-compose.yaml index a3b5de8c..5cf56c83 100644 --- a/src/pe_reports/docker-compose.yaml +++ b/src/pe_reports/docker-compose.yaml @@ -1,16 +1,15 @@ ---- -version: "3.8" +version: "3.9" services: - pe_reports_rabbitmq: - container_name: pe_reports_rabbitmq - hostname: pe_reports_rabbitmq + atc_rabbitmq: + container_name: atc_rabbitmq + hostname: atc_rabbitmq image: rabbitmq:3.8.14-management restart: always ports: - - 15672:15672 - - 5672:5672 + - 15674:15672 + - 5674:5672 env_file: - - ../../src/pe_reports/pe_reports_django_project/.env + - ./pe_reports_django_project/.env environment: - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER} - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASS} @@ -24,37 +23,70 @@ services: - LANGUAGE=C.UTF-8 - LC_ALL=C.UTF-8 volumes: - - /home/ubuntu/pe-reports/src/pe_reports/data/rabbitmq/data:/var/lib/rabbitmq/mnesia:rw - - /var/www/pe-reports/src/pe_reports/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + - ./pe_reports_django_project/data/rabbitmq/data:/var/lib/rabbitmq/mnesia:rw + - ./pe_reports_django_project/config:/etc/rabbitmq:rw networks: - - pe_reports_rabbitmq_network + - atc_network - pe_reports_redis: - container_name: pe_reports_redis - hostname: pe_reports_redis - image: redis:latest + atc_redis: + container_name: atc_redis + hostname: atc_redis + image: redis:7.4.1 restart: always ports: - - 6379:6379 + - 6378:6379 volumes: - - redis_data:/data + - ./pe_reports_django_project/redis_data:/data networks: - - pe_reports_redis_network + - atc_network + + web: + build: . + container_name: atc_web + volumes: + - ./pe_reports_django_project:/code + - ./pe_reports_django_project/config:/code/config + ports: + - "8002:8000" + env_file: + - pe_reports_django_project/.env + environment: + - DJANGO_SETTINGS_MODULE:pe_reports_django.settings + networks: + - atc_network + + nginx: image: nginx:1.25.0 + container_name: atc_nginx ports: - - "8089:8089" + - "8091:8091" volumes: - - ./config/nginx_config_conf.d:/etc/nginx/conf.d - - ./pe_reports_django_project/static:/var/www/pe-reports/static - - ./pe-reports:/var/www/pe-reports + - ./pe_reports_django_project/config/nginx_config_conf.d:/etc/nginx/conf.d networks: - - pe_reports_nginx_network + - atc_network + depends_on: + - web + + # database: + # image: postgres + # restart: always + # env_file: + # - ./pe_reports_django_project/.env + # networks: + # - atc_network + # volumes: + # - ./pe_reports_django_project/postgres_data:/var/lib/postgresql/data + # ports: + # - 5437:5432 + # container_name: atc_database networks: - pe_reports_network: + atc_network: driver: bridge + + volumes: - redis_data: {} + redis_data: {} \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/.env b/src/pe_reports/pe_reports_django_project/.env index 65d40d47..b46d7f3a 100644 --- a/src/pe_reports/pe_reports_django_project/.env +++ b/src/pe_reports/pe_reports_django_project/.env @@ -5,6 +5,13 @@ password= host= port= +# Mini Data Lake +mdl_host= +mdl_database= +mdl_user= +mdl_password= +mdl_port= + # The following key is for PE Service API_KEY= USER_REFRESH_TOKEN= diff --git a/src/pe_reports/pe_reports_django_project/config/nginx_config_conf.d/nginx.conf b/src/pe_reports/pe_reports_django_project/config/nginx_config_conf.d/nginx.conf new file mode 100755 index 00000000..6008eb2c --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/config/nginx_config_conf.d/nginx.conf @@ -0,0 +1,12 @@ +server { + listen 8091; + server_name localhost; + + location / { + proxy_pass http://web:8000; # Assuming 'web' is the service name and '8000' is the port where Gunicorn runs + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/config/rabbitmq.conf b/src/pe_reports/pe_reports_django_project/config/rabbitmq.conf new file mode 100755 index 00000000..9c184353 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/config/rabbitmq.conf @@ -0,0 +1,4 @@ +loopback_users.guest = false +listeners.tcp.default = 5672 +default_pass = guest1 +default_user = admin diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq-feature_flags b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq-feature_flags new file mode 100755 index 00000000..a9a883dd --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq-feature_flags @@ -0,0 +1,2 @@ +[implicit_default_bindings,maintenance_mode_status,quorum_queue,user_limits, + virtual_host_metadata]. diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq.pid b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq.pid new file mode 100644 index 00000000..88101bcc --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq.pid @@ -0,0 +1 @@ +465 \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/cluster_nodes.config b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/cluster_nodes.config new file mode 100644 index 00000000..48a2fbb4 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/cluster_nodes.config @@ -0,0 +1 @@ +{[rabbit@atc_rabbitmq],[rabbit@atc_rabbitmq]}. diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost new file mode 100755 index 00000000..35ec3b9d --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/.vhost @@ -0,0 +1 @@ +/ \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_persistent/0.rdq b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_persistent/0.rdq new file mode 100755 index 00000000..e69de29b diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_transient/0.rdq b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/msg_store_transient/0.rdq new file mode 100644 index 00000000..e69de29b diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/recovery.dets b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/recovery.dets new file mode 100755 index 00000000..c1ae47e3 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/msg_stores/vhosts/628WB79CIFDYO9LJI6DKMI09L/recovery.dets differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/nodes_running_at_shutdown b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/nodes_running_at_shutdown new file mode 100644 index 00000000..17ea9a5f --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/nodes_running_at_shutdown @@ -0,0 +1 @@ +[rabbit@atc_rabbitmq]. diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/quorum/rabbit@atc_rabbitmq/meta.dets b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/quorum/rabbit@atc_rabbitmq/meta.dets new file mode 100755 index 00000000..c1ae47e3 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/quorum/rabbit@atc_rabbitmq/meta.dets differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/quorum/rabbit@atc_rabbitmq/names.dets b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/quorum/rabbit@atc_rabbitmq/names.dets new file mode 100755 index 00000000..c1ae47e3 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/quorum/rabbit@atc_rabbitmq/names.dets differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_exchange.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_exchange.DCD new file mode 100755 index 00000000..abdefaa3 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_exchange.DCD differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_queue.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_queue.DCD new file mode 100755 index 00000000..37a87128 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_queue.DCD differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_route.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_route.DCD new file mode 100755 index 00000000..f8dd237a --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_durable_route.DCD @@ -0,0 +1 @@ +cXM \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_runtime_parameters.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_runtime_parameters.DCD new file mode 100755 index 00000000..15854308 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_runtime_parameters.DCD differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_serial b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_serial new file mode 100644 index 00000000..40c58719 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_serial @@ -0,0 +1 @@ +26. diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_topic_permission.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_topic_permission.DCD new file mode 100755 index 00000000..f8dd237a --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_topic_permission.DCD @@ -0,0 +1 @@ +cXM \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user.DCD new file mode 100755 index 00000000..87835539 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user.DCD differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user_permission.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user_permission.DCD new file mode 100755 index 00000000..a58a4453 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_user_permission.DCD differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_vhost.DCD b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_vhost.DCD new file mode 100755 index 00000000..21746957 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/rabbit_vhost.DCD differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema.DAT b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema.DAT new file mode 100755 index 00000000..fb31f735 Binary files /dev/null and b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema.DAT differ diff --git a/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema_version b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema_version new file mode 100755 index 00000000..203a50c1 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/data/rabbitmq/data/rabbit@atc_rabbitmq/schema_version @@ -0,0 +1 @@ +[store_msg,persistent_bytes,multiple_routing_keys,exchange_options,queue_options,topic_permission,vhost_limits,user_password_hashing,cluster_name,policy_apply_to,topic_trie_node,mirrored_supervisor,gm,user_admin_to_tags,exchange_event_serial,semi_durable_route,topic_trie,add_opts_to_listener,remove_user_scope,move_messages_to_vhost_store]. diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py index dc0ec933..bfc502ab 100644 --- a/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/schemas.py @@ -31,6 +31,17 @@ class Config: orm_mode = True +class GenInputOrgName(BaseModel): + """GenInputOrgUIDList schema class.""" + + org_acronym: str + + class Config: + """GenInputOrgUIDList config.""" + + orm_mode = True + + class OrganizationBase(BaseModel): """OrganizationBase schema.""" @@ -3039,6 +3050,7 @@ class XpanseBusinessUnitsInsert(BaseModel): entity_type: Optional[str] = None region: Optional[str] = None rating: Optional[int] = None + cyhy_db_name: Optional[str] = None # --- xpanse endpoint, Issue 682 --- @@ -3661,7 +3673,7 @@ class OrganizationsFullTable(BaseModel): class Config: """OrganizationsFullTable schema config class.""" - + orm_mode = True @@ -3758,7 +3770,6 @@ class Config: orm_mode = True - # --- domain_permu_insert_dnstwist(), Issue 706 pe-reports/005 atc-framework --- # Insert multiple dnstwist records into the domain_permutations table class DomainPermuInsertDNSTwist(BaseModel): @@ -3802,12 +3813,12 @@ class Config: # --- get_root_domains(), Issue 707 pe-reports/006 atc-framework --- # This function will reuse the schemas from the /rootdomains_by_org_uid endpoint - + # --- getDataSource(), Issue 708 pe-reports/007 atc-framework --- # This function will reuse the schemas from the /data_source_by_name endpoint - + # --- execute_hibp_breach_values(), Issue 709 pe-reports/008 atc-framework --- # Insert bulk HIBP breach data into credential_breaches table class CredBreachesHIBPInsert(BaseModel): @@ -3879,7 +3890,7 @@ class Config: """CredExpHIBPInsertInput schema config class.""" orm_mode = True - + # --- get_breach_uids(), Issue 010 atc-framework --- # Retrieve all breach names and uids @@ -3897,7 +3908,7 @@ class Config: # --- query_orgs(), Issue 011 atc-framework --- # Reuses OrganizationsFullTable schema - + # --- query_PE_subs(), Issue 012 atc-framework --- class SubdomainsByOrgUIDInput(BaseModel): @@ -3922,22 +3933,28 @@ class Config: """SubdomainsByOrgUID schema config class.""" orm_mode = True - + # --- insert_shodan_assets(), Issue 016 atc-framework --- # Insert bulk Shodan data into shodan_assets table class ShodanAssetsInsert(BaseModel): """ShodanAssetsInsert schema class.""" - email: Optional[str] = None + asn: Optional[int] = None + domains: Optional[List[str]] = None + hostnames: Optional[List[str]] = None + ip: Optional[str] = None + isn: Optional[str] = None + organization: Optional[str] = None organizations_uid: Optional[str] = None - root_domain: Optional[str] = None - sub_domain: Optional[str] = None - modified_date: Optional[str] = None - breach_name: Optional[str] = None - credential_breaches_uid: Optional[str] = None + port: Optional[int] = None + product: Optional[str] = None + protocol: Optional[str] = None + tags: Optional[List[str]] = None + timestamp: Optional[str] = None + country_code: Optional[str] = None + location: Optional[str] = None data_source_uid: Optional[str] = None - name: Optional[str] = None class Config: """ShodanAssetsInsert schema config class.""" @@ -3963,15 +3980,42 @@ class Config: class ShodanVulnsInsert(BaseModel): """ShodanVulnsInsert schema class.""" - email: Optional[str] = None organizations_uid: Optional[str] = None - root_domain: Optional[str] = None - sub_domain: Optional[str] = None - modified_date: Optional[str] = None - breach_name: Optional[str] = None - credential_breaches_uid: Optional[str] = None + organization: Optional[str] = None + ip: Optional[str] = None + port: Optional[str] = None + protocol: Optional[str] = None + timestamp: Optional[str] = None + cve: Optional[str] = None + severity: Optional[str] = None + cvss: Optional[float] = None + summary: Optional[str] = None + product: Optional[str] = None + attack_vector: Optional[str] = None + av_description: Optional[str] = None + attack_complexity: Optional[str] = None + ac_description: Optional[str] = None + confidentiality_impact: Optional[str] = None + ci_description: Optional[str] = None + integrity_impact: Optional[str] = None + ii_description: Optional[str] = None + availability_impact: Optional[str] = None + ai_description: Optional[str] = None + tags: Optional[List[str]] = None + domains: Optional[List[str]] = None + hostnames: Optional[List[str]] = None + isn: Optional[str] = None + asn: Optional[int] = None data_source_uid: Optional[str] = None + type: Optional[str] = None name: Optional[str] = None + potential_vulns: Optional[List[str]] = None + mitigation: Optional[str] = None + server: Optional[str] = None + is_verified: Optional[bool] = None + banner: Optional[str] = None + version: Optional[str] = None + cpe: Optional[List[str]] = None class Config: """ShodanVulnsInsert schema config class.""" @@ -3984,7 +4028,7 @@ class Config: class ShodanVulnsInsertInput(BaseModel): """ShodanVulnsInsertInput schema class.""" - vulns_data: List[ShodanVulnsInsert] + vuln_data: List[ShodanVulnsInsert] class Config: """ShodanVulnsInsertInput schema config class.""" @@ -3995,6 +4039,7 @@ class Config: # --- get_demo_orgs(), Issue 018 atc-framework --- # Reuses OrganizationsFullTable schema + class OrgsAssetsPagedInput(BaseModel): """OrgsAssetsPagedInput schema class.""" @@ -4006,9 +4051,10 @@ class Config: orm_mode = True + class OrgsWithAssets(BaseModel): """OrgsWithAssets schema class.""" - + org_name: Optional[str] = None acronym: Optional[str] = None retired: Optional[bool] = None @@ -4031,7 +4077,6 @@ class OrgsWithAssets(BaseModel): parent_acronym: Optional[str] = None networks: Optional[List[str]] = None root_domains: Optional[List[str]] = None - class Config: """OrgsWithAssets schema config class.""" @@ -4039,7 +4084,7 @@ class Config: orm_mode = True validate_assignment = True - + class OrgAssetPagedResult(BaseModel): """OrgAssetPagedResult schema class.""" @@ -4047,10 +4092,164 @@ class OrgAssetPagedResult(BaseModel): current_page: int data: List[OrgsWithAssets] + class OrgAssetTaskResp(BaseModel): """OrgAssetTaskResp schema class.""" task_id: str status: str result: Optional[OrgAssetPagedResult] = None + error: Optional[str] = None + + +class WasFindingInsert(BaseModel): + """WasFindingResult schema class""" + finding_uid: str + finding_type: Optional[str] = None + webapp_id: Optional[str] = None + was_org_id: Optional[str] = None + owasp_category: Optional[str] = None + severity: Optional[str] = None + times_detected: Optional[int] = None + base_score: Optional[float] = None + temporal_score: Optional[float] = None + fstatus: Optional[str] = None + last_detected: Optional[str] = None + first_detected: Optional[str] = None + is_remediated: Optional[bool] = None + potential: Optional[bool] = None + webapp_url: Optional[str] = None + webapp_name: Optional[str] = None + name: Optional[str] = None + cvss_v3_attack_vector: Optional[str] = None + cwe_list: Optional[List[str]] = None + wasc_list: Optional[List[Dict]] = None + last_tested: Optional[str] = None + fixed_date: Optional[str] = None + is_ignored: Optional[bool] = None + url: Optional[str] = None + qid: Optional[int] = None + response: Optional[str] = None + +class WasFindingResult(BaseModel): + """WasFindingResult schema class""" + finding_uid: str + finding_type: Optional[str] = None + webapp_id: Optional[str] = None + was_org_id: Optional[str] = None + owasp_category: Optional[str] = None + severity: Optional[str] = None + times_detected: Optional[int] = None + base_score: Optional[float] = None + temporal_score: Optional[float] = None + fstatus: Optional[str] = None + last_detected: Optional[str] = None + first_detected: Optional[str] = None + is_remediated: Optional[bool] = None + potential: Optional[bool] = None + webapp_url: Optional[str] = None + webapp_name: Optional[str] = None + name: Optional[str] = None + cvss_v3_attack_vector: Optional[str] = None + cwe_list: Optional[List[str]] = None + wasc_list: Optional[List[Dict]] = None + last_tested: Optional[str] = None + fixed_date: Optional[str] = None + is_ignored: Optional[bool] = None + url: Optional[str] = None + qid: Optional[int] = None + response: Optional[str] = None + +class WasFindingPagedResp(BaseModel): + """OrgAssetPagedResult schema class.""" + + total_pages: int + current_page: int + data: Optional[List[WasFindingResult]] = None + +class WasFindingsTaskResp(BaseModel): + """WasFindingsTaskResp schema class.""" + task_id: str + status: str + result: Optional[WasFindingPagedResp] = None + error: Optional[str] = None + + +class WasFindingsPagedInput(BaseModel): + """WasFindingsPagedInput schema class.""" + + org_acronym: str + page: int + per_page: int + + class Config: + """WasFindingsPagedInput schema config class.""" + + orm_mode = True + + + +class WasReportInsert(BaseModel): + """WasReportInsert schema class.""" + + org_name: Optional[str] = Field(default=None) + date_pulled: Optional[str] = Field(default=None) + last_scan_date: Optional[str] = Field(default=None) + security_risk: Optional[str] = Field(default=None) + total_info: Optional[int] = Field(default=None) + num_apps: Optional[int] = Field(default=None) + risk_color: Optional[str] = Field(default=None) + sensitive_count: Optional[int] = Field(default=None) + sensitive_color: Optional[str] = Field(default=None) + max_days_open_urgent: Optional[int] = Field(default=None) + max_days_open_critical: Optional[int] = Field(default=None) + urgent_color: Optional[str] = Field(default=None) + critical_color: Optional[str] = Field(default=None) + org_was_acronym: Optional[str] = Field(default=None) + name_len: Optional[str] = Field(default=None) + vuln_csv_dict: Optional[Dict] = Field(default={}) + ssn_cc_dict: Optional[Dict] = Field(default={}) + app_overview_csv_dict: Optional[Dict] = Field(default={}) + details_csv: Optional[List] = Field(default=[]) + info_csv: Optional[List] = Field(default=[]) + links_crawled: Optional[List] = Field(default=[]) + links_rejected: Optional[List] = Field(default=[]) + emails_found: Optional[List] = Field(default=[]) + owasp_count_dict: Optional[Dict] = Field(default={}) + group_count_dict: Optional[Dict] = Field(default={}) + fixed: Optional[int] = Field(default=None) + total: Optional[int] = Field(default=None) + vulns_monthly_dict: Optional[Dict] = Field(default={}) + path_disc: Optional[int] = Field(default=None) + info_disc: Optional[int] = Field(default=None) + cross_site: Optional[int] = Field(default=None) + burp: Optional[int] = Field(default=None) + sql_inj: Optional[int] = Field(default=None) + bugcrowd: Optional[int] = Field(default=None) + reopened: Optional[int] = Field(default=None) + reopened_color: Optional[str] = Field(default=None) + new_vulns: Optional[int] = Field(default=None) + new_vulns_color: Optional[str] = Field(default=None) + tot_vulns: Optional[int] = Field(default=None) + tot_vulns_color: Optional[str] = Field(default=None) + lev1: Optional[int] = Field(default=None) + lev2: Optional[int] = Field(default=None) + lev3: Optional[int] = Field(default=None) + lev4: Optional[int] = Field(default=None) + lev5: Optional[int] = Field(default=None) + severities: Optional[List[int]] = Field(default=[]) + ages: Optional[List[int]] = Field(default=[]) + pdf_obj: Optional[str] = Field(default=None) + + class Config: + """WasReportInsert schema config class.""" + + orm_mode = True + + +class WasReportTaskResp(BaseModel): + """WasReportTaskResp schema class.""" + task_id: str + status: str + result: Optional[WasReportInsert] = None error: Optional[str] = None \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py b/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py old mode 100644 new mode 100755 index b4bf133a..b7c25ca2 --- a/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/tasks.py @@ -60,8 +60,16 @@ VwShodanvulnsSuspected, VwShodanvulnsVerified, WasFindings, + WasReport, XpanseAlerts, ) +from dmz_mini_dl.models import ( + CredentialBreaches as MDL_CredentialBreaches, + CredentialExposures as MDL_CredentialExposures, + DataSource as MDL_DataSource, + Organization as MDL_Organization, + WasReport as MDL_WasReport +) # cisagov Libraries from pe_reports.helpers import ip_passthrough @@ -69,7 +77,7 @@ # Import schemas from . import schemas -LOGGER = logginng.getLogger(__name__) +LOGGER = logging.getLogger(__name__) # ---------- Task Helper Functions ---------- def convert_uuid_to_string(uuid): @@ -1223,8 +1231,31 @@ def cred_breach_sixgill_task(self, new_breaches: List[dict]): """Task function for the cred_breaches_sixgill_insert API endpoint.""" create_ct = 0 update_ct = 0 + try: + mdl_source_inst = MDL_DataSource.objects.get(name="Sixgill") + except MDL_DataSource.DoesNotExist: + LOGGER.warning(f"DataSource Sixgill not found.") + mdl_source_inst = None # Set to None if DataSource is not found for new_breach in new_breaches: # Insert each row of data + try: + MDL_CredentialBreaches.objects.get(breach_name=new_breach["breach_name"]) + + MDL_CredentialBreaches.objects.filter(breach_name=new_breach["breach_name"] + ).update( + password_included=new_breach["password_included"], + ) + except MDL_CredentialBreaches.DoesNotExist: + MDL_CredentialBreaches.objects.create( + credential_breaches_uid=uuid.uuid1(), + breach_name=new_breach["breach_name"], + description=new_breach["description"], + breach_date=new_breach["breach_date"], + password_included=new_breach["password_included"], + data_source=mdl_source_inst, + modified_date=new_breach["modified_date"], + ) + try: CredentialBreaches.objects.get(breach_name=new_breach["breach_name"]) # If record already exists, update @@ -1264,7 +1295,43 @@ def cred_exp_sixgill_task(self, new_exposures: List[dict]): """Task function for the credexp_insert API endpoint.""" update_ct = 0 create_ct = 0 + try: + mdl_source_inst = MDL_DataSource.objects.get(name="Sixgill") + except MDL_DataSource.DoesNotExist: + LOGGER.warning(f"DataSource Sixgill not found.") + mdl_source_inst = None # Set to None if DataSource is not found for new_exposure in new_exposures: + curr_org_inst = Organizations.objects.get( + organizations_uid=new_exposure["organizations_uid"] + ) + try: + MDL_CredentialExposures.objects.get( + breach_name=new_exposure["breach_name"], + email=new_exposure["email"], + ) + except MDL_CredentialExposures.DoesNotExist: + mdl_org_inst = MDL_Organization.objects.get( + acronym=curr_org_inst.cyhy_db_name + ) + + mdl_breach_inst = CredentialBreaches.objects.get( + breach_name=new_exposure["breach_name"] + ) + CredentialExposures.objects.create( + credential_exposures_uid=uuid.uuid1(), + modified_date=new_exposure["modified_date"], + sub_domain=new_exposure["sub_domain"], + email=new_exposure["email"], + hash_type=new_exposure["hash_type"], + name=new_exposure["name"], + login_id=new_exposure["login_id"], + password=new_exposure["password"], + phone=new_exposure["phone"], + breach_name=new_exposure["breach_name"], + organization=mdl_org_inst, + data_source=mdl_source_inst, + credential_breaches=mdl_breach_inst, + ) try: CredentialExposures.objects.get( breach_name=new_exposure["breach_name"], @@ -1272,9 +1339,7 @@ def cred_exp_sixgill_task(self, new_exposures: List[dict]): ) except CredentialExposures.DoesNotExist: # If cred exp record doesn't exist yet, create one - curr_org_inst = Organizations.objects.get( - organizations_uid=new_exposure["organizations_uid"] - ) + curr_source_inst = DataSource.objects.get( data_source_uid=new_exposure["data_source_uid"] ) @@ -1658,4 +1723,96 @@ def get_orgs_and_assets(self, page: int, per_page: int): "current_page": page, "data": orgs_list, } - return result \ No newline at end of file + return result + +def base64_to_bytes(base64_str) -> bytes: + """Convert a Base64-encoded string to binary data.""" + if base64_str: + return base64.b64decode(base64_str) + else: + return base64_str + +@shared_task(bind=True) +def insert_was_report(self, data): + # try: + LOGGER.info('Top of Was Report Task') + LOGGER.info(data) + defaults_dict={ + "org_name": data.get('org_name'), + "date_pulled": data.get('date_pulled'), + "last_scan_date": data.get('last_scan_date'), + "security_risk": data.get('security_risk'), + "total_info": data.get('total_info'), + "num_apps": data.get('num_apps'), + "risk_color": data.get('risk_color'), + "sensitive_count": data.get('sensitive_count'), + "sensitive_color": data.get('sensitive_color'), + "max_days_open_urgent": data.get('max_days_open_urgent'), + "max_days_open_critical": data.get('max_days_open_critical'), + "urgent_color": data.get('urgent_color'), + "critical_color": data.get('critical_color'), + "name_len": data.get('name_len'), + "vuln_csv_dict": data.get('vuln_csv_dict'), + "ssn_cc_dict": data.get('ssn_cc_dict'), + "app_overview_csv_dict": data.get('app_overview_csv_dict'), + "details_csv": data.get('details_csv'), + "info_csv": data.get('info_csv'), + "links_crawled": data.get('links_crawled'), + "links_rejected": data.get('links_rejected'), + "emails_found": data.get('emails_found'), + "owasp_count_dict": data.get('owasp_count_dict'), + "group_count_dict": data.get('group_count_dict'), + "fixed": data.get('fixed'), + "total": data.get('total'), + "vulns_monthly_dict": data.get('vulns_monthly_dict'), + "path_disc": data.get('path_disc'), + "info_disc": data.get('info_disc'), + "cross_site": data.get('cross_site'), + "burp": data.get('burp'), + "sql_inj": data.get('sql_inj'), + "bugcrowd": data.get('bugcrowd'), + "reopened": data.get('reopened'), + "reopened_color": data.get('reopened_color'), + "new_vulns": data.get('new_vulns'), + "new_vulns_color": data.get('new_vulns_color'), + "tot_vulns": data.get('tot_vulns'), + "tot_vulns_color": data.get('tot_vulns_color'), + "lev1": data.get('lev1'), + "lev2": data.get('lev2'), + "lev3": data.get('lev3'), + "lev4": data.get('lev4'), + "lev5": data.get('lev5'), + "severities": data.get('severities'), + "ages": data.get('ages'), + "pdf_obj": base64_to_bytes(data.get('pdf_obj')) + + } + was_report_object, created = WasReport.objects.update_or_create( + org_was_acronym = data.get('org_was_acronym'), + last_scan_date = data.get('last_scan_date'), + defaults=defaults_dict + ) + try: + mdl_was_report_object, mdl_created = MDL_WasReport.objects.update_or_create( + org_was_acronym = data.get('org_was_acronym'), + last_scan_date = data.get('last_scan_date'), + defaults=defaults_dict + ) + except Exception: + LOGGER.info(f"Failed to insert WAS report for {data.get('org_was_acronym')}") + + + if created: + LOGGER.info("New Was record created for %s", data.get('org_was_acronym')) + return { + "message": "New Was record created.", + "was_report_id": was_report_object.id, + } + else: + return {"message": "Record updated successfully.", "was_report_id": was_report_object.id} + + # except Exception as e: + # LOGGER.error(e) + # print("failed to insert or update") + # return {"message": "Failed to insert or update.", "was_report_obj": None, "error": e} + # LOGGER.info("API key expired please try again") \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/dataAPI/views.py b/src/pe_reports/pe_reports_django_project/dataAPI/views.py index 2ee07eb1..a623ea82 100644 --- a/src/pe_reports/pe_reports_django_project/dataAPI/views.py +++ b/src/pe_reports/pe_reports_django_project/dataAPI/views.py @@ -42,6 +42,7 @@ get_kev_list_info, get_l_stakeholders_info, get_m_stakeholders_info, + get_orgs_and_assets, get_s_stakeholders_info, get_ve_info, get_vs_info, @@ -61,7 +62,8 @@ sub_domains_by_org_task, sub_domains_table_task, top_cves_insert_task, - was_vulns_task + was_vulns_task, + insert_was_report ) from decouple import config from django.conf import settings @@ -70,6 +72,21 @@ from django.db import transaction from django.db.models import F, Q from django.forms.models import model_to_dict +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from dmz_mini_dl.models import CredentialBreaches as MDL_CredentialBreaches +from dmz_mini_dl.models import CredentialExposures as MDL_CredentialExposures +from dmz_mini_dl.models import DataSource as MDL_DataSource +from dmz_mini_dl.models import Organization as MDL_Organization +from dmz_mini_dl.models import ShodanAssets as MDL_ShodanAssets +from dmz_mini_dl.models import ShodanVulns as MDL_ShodanVulns +from dmz_mini_dl.models import XpanseAlerts as MDL_XpanseAlerts +from dmz_mini_dl.models import XpanseAssetsMdl as MDL_XpanseAssets +from dmz_mini_dl.models import XpanseBusinessUnits as MDL_XpanseBusinessUnits +from dmz_mini_dl.models import XpanseCveServiceMdl as MDL_XpanseCveService +from dmz_mini_dl.models import XpanseCvesMdl as MDL_XpanseCves +from dmz_mini_dl.models import XpanseServicesMdl as MDL_XpanseServices +from dmz_mini_dl.models import WasFindings as MDL_WasFindings from fastapi import ( APIRouter, Depends, @@ -99,12 +116,14 @@ DataSource, DomainAlerts, DomainPermutations, + Ips, Mentions, Organizations, PshttResults, ReportSummaryStats, RootDomains, ShodanAssets, + ShodanVulns, SubDomains, VwBreachcomp, VwBreachcompBreachdetails, @@ -126,6 +145,8 @@ VwShodanvulnsSuspected, VwShodanvulnsVerified, WasTrackerCustomerdata, + WasFindings, + WasReport, WeeklyStatuses, XpanseAlerts, XpanseAssets, @@ -252,6 +273,7 @@ def userapiTokenUpdate(expiredaccessToken, user_refresh, theapiKey, user_id): def userapiTokenverify(theapiKey): """Check to see if api key is expired.""" tokenRecords = list(apiUser.objects.filter(apiKey=theapiKey)) + LOGGER.info(f"The user provided key is {theapiKey}") user_key = "" user_refresh = "" user_id = "" @@ -4234,8 +4256,7 @@ def rss_insert(data: schemas.RSSInsertInput, tokens: dict = Depends(get_api_key) try: # Check if record already exists ReportSummaryStats.objects.get( - organizations_uid=specified_org_uid, - start_date=data.start_date + organizations_uid=specified_org_uid, start_date=data.start_date ) # If it already exists, update ReportSummaryStats.objects.filter( @@ -4880,7 +4901,7 @@ def cred_breach_intelx( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, create task for query task = cred_breach_intelx_task.delay(data.source_uid) # Return the new task id w/ "Processing" status @@ -5586,13 +5607,38 @@ def cred_breaches_intelx_insert( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, insert intelx breach data insert_count = 0 update_count = 0 + + try: + mdl_source_inst = MDL_DataSource.objects.get(name="IntelX") + except MDL_DataSource.DoesNotExist: + LOGGER.warning("DataSource IntelX not found.") + mdl_source_inst = None # Set to None if DataSource is not found + for row in data.breach_data: # Check if record already exists row_dict = row.__dict__ + try: + MDL_CredentialBreaches.objects.update_or_create( + breach_name=row_dict["breach_name"], + defaults={ + "description": row_dict["description"], + "breach_date": row_dict["breach_date"], + "added_date": row_dict["added_date"], + "modified_date": timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), + "password_included": row_dict["password_included"], + "data_source": mdl_source_inst, + }, + ) + except Exception: + LOGGER.warn('Failed to save or update Credential Breach to MDL.') + breach_results = CredentialBreaches.objects.filter( breach_name=row_dict["breach_name"] ) @@ -5606,7 +5652,10 @@ def cred_breaches_intelx_insert( description=row_dict["description"], breach_date=row_dict["breach_date"], added_date=row_dict["added_date"], - modified_date=row_dict["modified_date"], + modified_date=timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), password_included=row_dict["password_included"], data_source_uid=curr_data_source_inst, ) @@ -5622,8 +5671,10 @@ def cred_breaches_intelx_insert( + str(update_count) + " records updated in the credential_breaches table" ) - except ObjectDoesNotExist: - LOGGER.info("API key expired please try again") + # except ObjectDoesNotExist: + # LOGGER.info("API key expired please try again") + except Exception as e: + LOGGER.info(f"Error: {e}") else: return {"message": "No api key was submitted"} @@ -5644,12 +5695,20 @@ def cred_exp_intelx_insert( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, insert intelx data create_cnt = 0 update_cnt = 0 + try: + mdl_source_inst = MDL_DataSource.objects.get(name="IntelX") + except MDL_DataSource.DoesNotExist: + LOGGER.warning("DataSource with IntelX not found.") + mdl_source_inst = None # Set to None if DataSource is not found for row in data.exp_data: row_dict = row.__dict__ + curr_org_inst = Organizations.objects.get( + organizations_uid=row_dict["organizations_uid"] + ) try: CredentialExposures.objects.get( breach_name=row_dict["breach_name"], @@ -5663,9 +5722,7 @@ def cred_exp_intelx_insert( update_cnt += 1 except CredentialExposures.DoesNotExist: # If record doesn't exist yet, create one - curr_org_inst = Organizations.objects.get( - organizations_uid=row_dict["organizations_uid"] - ) + curr_source_inst = DataSource.objects.get( data_source_uid=row_dict["data_source_uid"] ) @@ -5679,7 +5736,10 @@ def cred_exp_intelx_insert( root_domain=row_dict["root_domain"], sub_domain=row_dict["sub_domain"], breach_name=row_dict["breach_name"], - modified_date=row_dict["modified_date"], + modified_date=timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), data_source_uid=curr_source_inst, password=row_dict["password"], hash_type=row_dict["hash_type"], @@ -5687,6 +5747,44 @@ def cred_exp_intelx_insert( credential_breaches_uid=curr_breach_inst, ) create_cnt += 1 + try: + MDL_CredentialExposures.objects.get( + breach_name=row_dict["breach_name"], + email=row_dict["email"], + ) + MDL_CredentialExposures.objects.filter( + breach_name=row_dict["breach_name"], + email=row_dict["email"], + ).update(modified_date=row_dict["modified_date"]) + + except MDL_CredentialExposures.DoesNotExist: + mdl_org_inst = MDL_Organization.objects.get( + acronym=curr_org_inst.cyhy_db_name + ) + # mdl_source_inst = MDL_DataSource.objects.get( + # name='IntelX' + # ) + + mdl_breach_inst = MDL_CredentialBreaches.objects.get( + breach_name=row_dict["breach_name"], + ) + MDL_CredentialExposures.objects.create( + # credential_exposures_uid=uuid.uuid1(), + email=row_dict["email"], + organization=mdl_org_inst, + root_domain=row_dict["root_domain"], + sub_domain=row_dict["sub_domain"], + breach_name=row_dict["breach_name"], + modified_date=timezone.make_aware( + parse_datetime(row_dict["modified_date"]), + timezone.timezone.utc, + ), + data_source=mdl_source_inst, + password=row_dict["password"], + hash_type=row_dict["hash_type"], + intelx_system_id=row_dict["intelx_system_id"], + credential_breaches=mdl_breach_inst, + ) # Return success message return ( str(create_cnt) @@ -5694,8 +5792,10 @@ def cred_exp_intelx_insert( + str(update_cnt) + " records updated in the credential_exposures table" ) - except ObjectDoesNotExist: - LOGGER.info("API key expired please try again") + # except ObjectDoesNotExist: + # LOGGER.info("API key expired please try again") + except Exception as e: + LOGGER.info(f"Error: {e}") else: return {"message": "No api key was submitted"} @@ -5705,7 +5805,7 @@ def cred_exp_intelx_insert( "/xpanse_business_unit_insert_or_update", dependencies=[Depends(get_api_key)], # response_model=Dict[schemas.PshttDataBase], - tags=["Update or insert CVE data from NIST"], + tags=["Update or insert Xpanse Business Unit"], ) # @transaction.atomic def xpanse_business_unit_insert_or_update( @@ -5716,23 +5816,63 @@ def xpanse_business_unit_insert_or_update( """Create API endpoint to create a record in database.""" if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) LOGGER.info(f"The api key submitted {tokens}") + mapped_org = None + mdl_mapped_org = None + # TODO: only for fceb for now, need to update for when we add more xpanse stakeholder + # match = re.search(r'\((.*?)\)', data.entity_name) + if data.cyhy_db_name is not None: + try: + mapped_org = Organizations.objects.get( + cyhy_db_name=data.cyhy_db_name + ) + except Organizations.DoesNotExist: + mapped_org = None + try: + mdl_mapped_org = MDL_Organization.objects.get( + acronym=data.cyhy_db_name + ) + except MDL_Organization.DoesNotExist: + mdl_mapped_org = None + defaults = { + "state": data.state, + "county": data.county, + "city": data.city, + "sector": data.sector, + "entity_type": data.entity_type, + "region": data.region, + "rating": data.rating, + "cyhy_db_name": mapped_org, + } + + mdl_defaults = { + "state": data.state, + "county": data.county, + "city": data.city, + "sector": data.sector, + "entity_type": data.entity_type, + "region": data.region, + "rating": data.rating, + "cyhy_db_name": mdl_mapped_org, + } + # if mapped_org is not None: + # defaults["cyhy_db_name"] = mapped_org + + ( + mdl_business_unit_object, + mdl_created, + ) = MDL_XpanseBusinessUnits.objects.update_or_create( + entity_name=data.entity_name, + defaults=mdl_defaults, + ) + ( business_unit_object, created, ) = XpanseBusinessUnits.objects.update_or_create( - entity_name=data.entity_name, - defaults={ - "state": data.state, - "county": data.county, - "city": data.city, - "sector": data.sector, - "entity_type": data.entity_type, - "region": data.region, - "rating": data.rating, - }, + entity_name=data.entity_name, defaults=defaults ) if created: LOGGER.info( @@ -5754,12 +5894,43 @@ def xpanse_business_unit_insert_or_update( return {"message": "No api key was submitted"} +@api_router.get( + "/linked_xpanse_business_units", + dependencies=[ + Depends(get_api_key) + ], # Depends(RateLimiter(times=200, seconds=60))], + # response_model=List[schemas.OrganizationsFullTable], + tags=["Retrieve all Xpanse business units that link to an organization."], +) +def linked_xpanse_business_units(tokens: dict = Depends(get_api_key)): + """Call API endpoint to get Xpanse business unit that link to an organization.""" + # Check for API key + LOGGER.info(f"The api key submitted {tokens}") + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + # If API key valid, make query + xpanse_business_units = list( + XpanseBusinessUnits.objects.filter(cyhy_db_name__isnull=False).values() + ) + # Convert data types to match response model + for row in xpanse_business_units: + row["xpanse_business_unit_uid"] = convert_uuid_to_string( + row["xpanse_business_unit_uid"] + ) + return xpanse_business_units + except ObjectDoesNotExist: + LOGGER.info("API key expired please try again") + else: + return {"message": "No api key was submitted"} + + # --- xpanse endpoint, Issue 682 --- @api_router.put( "/xpanse_alert_insert_or_update", dependencies=[Depends(get_api_key)], # response_model=Dict[schemas.PshttDataBase], - tags=["Update or insert CVE data from NIST"], + tags=["Update or insert alert data from Xpanse"], ) # @transaction.atomic def xpanse_alert_insert_or_update( @@ -5770,167 +5941,225 @@ def xpanse_alert_insert_or_update( """Create API endpoint to create a record in database.""" if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) LOGGER.info(f"The api key submitted {tokens}") LOGGER.info("Got into Xpanse Alert insert") # vender_prod_dict = data.vender_product + alert_defaults = { + "time_pulled_from_xpanse": data.time_pulled_from_xpanse, + # "alert_id": data.alert_id, + "detection_timestamp": data.detection_timestamp, + "alert_name": data.alert_name, + "description": data.description, + "host_name": data.host_name, + "alert_action": data.alert_action, + "action_pretty": data.action_pretty, + "action_country": data.action_country, + "action_remote_port": data.action_remote_port, + "starred": data.starred, + "external_id": data.external_id, + "related_external_id": data.related_external_id, + "alert_occurrence": data.alert_occurrence, + "severity": data.severity, + "matching_status": data.matching_status, + "local_insert_ts": data.local_insert_ts, + "last_modified_ts": data.last_modified_ts, + "case_id": data.case_id, + "event_timestamp": data.event_timestamp, + "alert_type": data.alert_type, + "resolution_status": data.resolution_status, + "resolution_comment": data.resolution_comment, + "tags": data.tags, + "last_observed": data.last_observed, + "country_codes": data.country_codes, + "cloud_providers": data.cloud_providers, + "ipv4_addresses": data.ipv4_addresses, + "domain_names": data.domain_names, + "service_ids": data.service_ids, + "website_ids": data.website_ids, + "asset_ids": data.asset_ids, + "certificate": data.certificate, + "port_protocol": data.port_protocol, + "attack_surface_rule_name": data.attack_surface_rule_name, + "remediation_guidance": data.remediation_guidance, + "asset_identifiers": data.asset_identifiers + # business_units: Optional[List[str]] = None + # services: Optional[List[XpanseService]] = None + # assets : Optional[List[XpanseAsset]] = None + } + + # Create alerts in both databases alert_object, created = XpanseAlerts.objects.update_or_create( alert_id=data.alert_id, - defaults={ - "time_pulled_from_xpanse": data.time_pulled_from_xpanse, - # "alert_id": data.alert_id, - "detection_timestamp": data.detection_timestamp, - "alert_name": data.alert_name, - "description": data.description, - "host_name": data.host_name, - "alert_action": data.alert_action, - "action_pretty": data.action_pretty, - "action_country": data.action_country, - "action_remote_port": data.action_remote_port, - "starred": data.starred, - "external_id": data.external_id, - "related_external_id": data.related_external_id, - "alert_occurrence": data.alert_occurrence, - "severity": data.severity, - "matching_status": data.matching_status, - "local_insert_ts": data.local_insert_ts, - "last_modified_ts": data.last_modified_ts, - "case_id": data.case_id, - "event_timestamp": data.event_timestamp, - "alert_type": data.alert_type, - "resolution_status": data.resolution_status, - "resolution_comment": data.resolution_comment, - "tags": data.tags, - "last_observed": data.last_observed, - "country_codes": data.country_codes, - "cloud_providers": data.cloud_providers, - "ipv4_addresses": data.ipv4_addresses, - "domain_names": data.domain_names, - "service_ids": data.service_ids, - "website_ids": data.website_ids, - "asset_ids": data.asset_ids, - "certificate": data.certificate, - "port_protocol": data.port_protocol, - "attack_surface_rule_name": data.attack_surface_rule_name, - "remediation_guidance": data.remediation_guidance, - "asset_identifiers": data.asset_identifiers - # business_units: Optional[List[str]] = None - # services: Optional[List[XpanseService]] = None - # assets : Optional[List[XpanseAsset]] = None - }, + defaults=alert_defaults, ) if created: LOGGER.info("new Xpanse alert record created for %s", data.alert_name) + ( + mdl_alert_object, + mdl_alert_created, + ) = MDL_XpanseAlerts.objects.update_or_create( + alert_id=data.alert_id, + defaults=alert_defaults, + ) + if mdl_alert_created: + LOGGER.info( + "new Xpanse alert record created in MDL for %s", data.alert_name + ) + business_unit_list = [] + mdl_business_unit_list = [] for b_u in data.business_units: business_unit_list.append( XpanseBusinessUnits.objects.get(entity_name=b_u) ) - + mdl_business_unit_list.append( + MDL_XpanseBusinessUnits.objects.get(entity_name=b_u) + ) alert_object.business_units.set(business_unit_list) + mdl_alert_object.business_units.set(mdl_business_unit_list) asset_list = [] + mdl_asset_list = [] for asset_data in data.assets: + asset_defaults = { + "asset_name": asset_data.asset_name, + "asset_type": asset_data.asset_type, + "last_observed": asset_data.last_observed, + "first_observed": asset_data.first_observed, + "externally_detected_providers": asset_data.externally_detected_providers, + "created": asset_data.created, + "ips": asset_data.ips, + "active_external_services_types": asset_data.active_external_services_types, + "domain": asset_data.domain, + "certificate_issuer": asset_data.certificate_issuer, + "certificate_algorithm": asset_data.certificate_algorithm, + "certificate_classifications": asset_data.certificate_classifications, + "resolves": asset_data.resolves, + # details + "top_level_asset_mapper_domain": asset_data.top_level_asset_mapper_domain, + "domain_asset_type": asset_data.domain_asset_type, + "is_paid_level_domain": asset_data.is_paid_level_domain, + "domain_details": asset_data.domain_details, + "dns_zone": asset_data.dns_zone, + "latest_sampled_ip": asset_data.latest_sampled_ip, + "recent_ips": asset_data.recent_ips, + "external_services": asset_data.external_services, + "externally_inferred_vulnerability_score": asset_data.externally_inferred_vulnerability_score, + "externally_inferred_cves": asset_data.externally_inferred_cves, + "explainers": asset_data.explainers, + "tags": asset_data.tags, + } asset_object, created = XpanseAssets.objects.update_or_create( asm_id=asset_data.asm_id, - defaults={ - "asset_name": asset_data.asset_name, - "asset_type": asset_data.asset_type, - "last_observed": asset_data.last_observed, - "first_observed": asset_data.first_observed, - "externally_detected_providers": asset_data.externally_detected_providers, - "created": asset_data.created, - "ips": asset_data.ips, - "active_external_services_types": asset_data.active_external_services_types, - "domain": asset_data.domain, - "certificate_issuer": asset_data.certificate_issuer, - "certificate_algorithm": asset_data.certificate_algorithm, - "certificate_classifications": asset_data.certificate_classifications, - "resolves": asset_data.resolves, - # details - "top_level_asset_mapper_domain": asset_data.top_level_asset_mapper_domain, - "domain_asset_type": asset_data.domain_asset_type, - "is_paid_level_domain": asset_data.is_paid_level_domain, - "domain_details": asset_data.domain_details, - "dns_zone": asset_data.dns_zone, - "latest_sampled_ip": asset_data.latest_sampled_ip, - "recent_ips": asset_data.recent_ips, - "external_services": asset_data.external_services, - "externally_inferred_vulnerability_score": asset_data.externally_inferred_vulnerability_score, - "externally_inferred_cves": asset_data.externally_inferred_cves, - "explainers": asset_data.explainers, - "tags": asset_data.tags, - }, + defaults=asset_defaults, ) asset_list.append(asset_object) + ( + mdl_asset_object, + mdl_asset_created, + ) = MDL_XpanseAssets.objects.update_or_create( + asm_id=asset_data.asm_id, + defaults=asset_defaults, + ) + mdl_asset_list.append(mdl_asset_object) alert_object.assets.set(asset_list) + mdl_alert_object.assets.set(mdl_asset_list) services_list = [] + mdl_services_list = [] for service_data in data.services: + service_defaults = { + "service_name": service_data.service_name, + "service_type": service_data.service_type, + "ip_address": service_data.ip_address, + "domain": service_data.domain, + "externally_detected_providers": service_data.externally_detected_providers, + "is_active": service_data.is_active, + "first_observed": service_data.first_observed, + "last_observed": service_data.last_observed, + "port": service_data.port, + "protocol": service_data.protocol, + "active_classifications": service_data.active_classifications, + "inactive_classifications": service_data.inactive_classifications, + "discovery_type": service_data.discovery_type, + "externally_inferred_vulnerability_score": service_data.externally_inferred_vulnerability_score, + "externally_inferred_cves": service_data.externally_inferred_cves, + "service_key": service_data.service_key, + "service_key_type": service_data.service_key_type, + } service_object, created = XpanseServices.objects.update_or_create( service_id=service_data.service_id, - defaults={ - "service_name": service_data.service_name, - "service_type": service_data.service_type, - "ip_address": service_data.ip_address, - "domain": service_data.domain, - "externally_detected_providers": service_data.externally_detected_providers, - "is_active": service_data.is_active, - "first_observed": service_data.first_observed, - "last_observed": service_data.last_observed, - "port": service_data.port, - "protocol": service_data.protocol, - "active_classifications": service_data.active_classifications, - "inactive_classifications": service_data.inactive_classifications, - "discovery_type": service_data.discovery_type, - "externally_inferred_vulnerability_score": service_data.externally_inferred_vulnerability_score, - "externally_inferred_cves": service_data.externally_inferred_cves, - "service_key": service_data.service_key, - "service_key_type": service_data.service_key_type, - }, + defaults=service_defaults, + ) + ( + mdl_service_object, + mdl_service_created, + ) = MDL_XpanseServices.objects.update_or_create( + service_id=service_data.service_id, + defaults=service_defaults, ) - LOGGER.info(service_data) + # LOGGER.info(service_data) if service_data.cves is not None: for cve_data, cve_match_data in service_data.cves: LOGGER.info(cve_data) LOGGER.info(cve_match_data) + cve_defaults = { + "cvss_score_v2": cve_data.cvss_score_v2, + "cve_severity_v2": cve_data.cve_severity_v2, + "cvss_score_v3": cve_data.cvss_score_v3, + "cve_severity_v3": cve_data.cve_severity_v3, + } cve_object, created = XpanseCves.objects.update_or_create( cve_id=cve_data.cve_id, - defaults={ - "cvss_score_v2": cve_data.cvss_score_v2, - "cve_severity_v2": cve_data.cve_severity_v2, - "cvss_score_v3": cve_data.cvss_score_v3, - "cve_severity_v3": cve_data.cve_severity_v3, - }, + defaults=cve_defaults, ) - + ( + mdl_cve_object, + mdl_cve_created, + ) = MDL_XpanseCves.objects.update_or_create( + cve_id=cve_data.cve_id, + defaults=cve_defaults, + ) + cve_service_default = { + "inferred_cve_match_type": cve_match_data.inferred_cve_match_type, + "product": cve_match_data.product, + "confidence": cve_match_data.confidence, + "vendor": cve_match_data.vendor, + "version_number": cve_match_data.version_number, + "activity_status": cve_match_data.activity_status, + "first_observed": cve_match_data.first_observed, + "last_observed": cve_match_data.last_observed, + } ( cve_match_object, created, ) = XpanseCveService.objects.update_or_create( xpanse_inferred_cve=cve_object, xpanse_service=service_object, - defaults={ - "inferred_cve_match_type": cve_match_data.inferred_cve_match_type, - "product": cve_match_data.product, - "confidence": cve_match_data.confidence, - "vendor": cve_match_data.vendor, - "version_number": cve_match_data.version_number, - "activity_status": cve_match_data.activity_status, - "first_observed": cve_match_data.first_observed, - "last_observed": cve_match_data.last_observed, - }, + defaults=cve_service_default, + ) + ( + mdl_cve_match_object, + mdl_cve_service_created, + ) = MDL_XpanseCveService.objects.update_or_create( + xpanse_inferred_cve=mdl_cve_object, + xpanse_service=mdl_service_object, + defaults=cve_service_default, ) services_list.append(service_object) + mdl_services_list.append(mdl_service_object) alert_object.services.set(services_list) - alert_object.save() + mdl_alert_object.services.set(mdl_services_list) + mdl_alert_object.save() + # for vender, product_list in vender_prod_dict.items(): # vender_obj, vender_created = CpeVender.objects.update_or_create( @@ -6314,6 +6543,7 @@ async def get_pshtt_domains_to_run_status( else: return {"message": "No api key was submitted"} + # --- get_orgs(), Issue 699 pe-reports --- @api_router.get( "/organizations_demo_or_report_on", @@ -6323,13 +6553,13 @@ async def get_pshtt_domains_to_run_status( response_model=List[schemas.OrganizationsFullTable], tags=["Retrieve data for all demo or report_on orgs."], ) -def organizations_demo(tokens: dict = Depends(get_api_key)): +def organizations_demo_or_report_on(tokens: dict = Depends(get_api_key)): """Call API endpoint to get data for all demo or report_on orgs.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, make query organizations_demo_or_report_on_data = list( Organizations.objects.filter(Q(demo=True) | Q(report_on=True)).values() @@ -6376,7 +6606,6 @@ def pshtt_result_update_or_insert( ) sub_domain_uid = SubDomains.objects.get(sub_domain_uid=data.sub_domain_uid) - pshtt_object, created = PshttResults.objects.update_or_create( sub_domain_uid=data.sub_domain_uid, organizations_uid=data.organizations_uid, @@ -6466,34 +6695,32 @@ def pshtt_result_update_or_insert( response_model=List[schemas.DataSourceFullTable], tags=["Retrieve data for specified data source name."], ) -def data_source_by_name(data: schemas.DataSourceByNameInput, tokens: dict = Depends(get_api_key)): +def data_source_by_name( + data: schemas.DataSourceByNameInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get data for specified data source name.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, make query data_source_by_name_data = list( DataSource.objects.filter(name=data.name).values() ) # also update data source record - today = dt.today().strftime("%Y-%m-%d") + today = datetime.today().strftime("%Y-%m-%d") DataSource.objects.filter(name=data.name).update(last_run=today) # Convert data types to match response model for row in data_source_by_name_data: - row["data_source_uid"] = convert_uuid_to_string( - row["data_source_uid"] - ) - row["last_run"] = convert_date_to_string( - row["last_run"] - ) + row["data_source_uid"] = convert_uuid_to_string(row["data_source_uid"]) + row["last_run"] = convert_date_to_string(row["last_run"]) return data_source_by_name_data except ObjectDoesNotExist: LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- get_breaches(), Issue 701 pe-reports --- @api_router.get( @@ -6513,7 +6740,9 @@ def breach_names_and_uids(tokens: dict = Depends(get_api_key)): userapiTokenverify(theapiKey=tokens) # If API key valid, make query breach_names_and_uids_data = list( - CredentialBreaches.objects.all().values("breach_name", "credential_breaches_uid") + CredentialBreaches.objects.all().values( + "breach_name", "credential_breaches_uid" + ) ) # Convert data types to match response model for row in breach_names_and_uids_data: @@ -6525,7 +6754,7 @@ def breach_names_and_uids(tokens: dict = Depends(get_api_key)): LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- getSubdomain(), Issue 702 pe-reports --- @api_router.post( @@ -6536,7 +6765,9 @@ def breach_names_and_uids(tokens: dict = Depends(get_api_key)): response_model=List[schemas.SubdomainUIDByDomain], tags=["Retrieve data for the specified subdomain."], ) -def subdomain_by_domain(data: schemas.SubdomainUIDByDomainInput, tokens: dict = Depends(get_api_key)): +def subdomain_by_domain( + data: schemas.SubdomainUIDByDomainInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get data for specified subdomain.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") @@ -6545,7 +6776,9 @@ def subdomain_by_domain(data: schemas.SubdomainUIDByDomainInput, tokens: dict = userapiTokenverify(theapiKey=tokens) # If API key valid, make query subdomain_by_domain_data = list( - SubDomains.objects.filter(sub_domain=data.domain).values("sub_domain_uid") + SubDomains.objects.filter(sub_domain=data.domain).values( + "sub_domain_uid" + ) ) # Convert data types to match response model for row in subdomain_by_domain_data: @@ -6555,7 +6788,7 @@ def subdomain_by_domain(data: schemas.SubdomainUIDByDomainInput, tokens: dict = LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- org_root_domains(), Issue 703 pe-reports --- @api_router.post( @@ -6566,25 +6799,30 @@ def subdomain_by_domain(data: schemas.SubdomainUIDByDomainInput, tokens: dict = response_model=List[schemas.RootDomainsTable], tags=["Retrieve root domains for the specified org uid."], ) -def rootdomains_by_org_uid(data: schemas.RootdomainsByOrgUIDInput, tokens: dict = Depends(get_api_key)): +def rootdomains_by_org_uid( + data: schemas.RootdomainsByOrgUIDInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get root domains for specified org uid.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, make query rootdomains_by_org_uid_data = list( RootDomains.objects.filter( - organizations_uid=data.org_uid, - enumerate_subs=True + organizations_uid=data.org_uid, enumerate_subs=True ).values() ) # Convert data types to match response model for row in rootdomains_by_org_uid_data: row["root_domain_uid"] = convert_uuid_to_string(row["root_domain_uid"]) - row["organizations_uid_id"] = convert_uuid_to_string(row["organizations_uid_id"]) - row["data_source_uid_id"] = convert_uuid_to_string(row["data_source_uid_id"]) + row["organizations_uid_id"] = convert_uuid_to_string( + row["organizations_uid_id"] + ) + row["data_source_uid_id"] = convert_uuid_to_string( + row["data_source_uid_id"] + ) return rootdomains_by_org_uid_data except ObjectDoesNotExist: LOGGER.info("API key expired please try again") @@ -6595,25 +6833,22 @@ def rootdomains_by_org_uid(data: schemas.RootdomainsByOrgUIDInput, tokens: dict @api_router.post( "/crossfeed_vulns", dependencies=[Depends(get_api_key)], - #response_model=schemas.PshttDomainToRunTaskResp,TODO, create schema for generlized output + # response_model=schemas.PshttDomainToRunTaskResp,TODO, create schema for generlized output tags=["Return all vulnerabilites formatted for crossfeed database."], ) -def crossfeed_vulns( - data: schemas.GenInputOrgName, - tokens: dict = Depends(get_api_key) - ): +def crossfeed_vulns(data: schemas.GenInputOrgName, tokens: dict = Depends(get_api_key)): """Returna all vulnerabilities for crossfeed database.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") if tokens: tasks_dict = {} shodan_task = shodan_vulns_task.delay(data.org_acronym) - tasks_dict['shodan'] = shodan_task.id + tasks_dict["shodan"] = shodan_task.id cred_task = credential_breach_vulns_task.delay(data.org_acronym) - tasks_dict['creds'] = cred_task.id + tasks_dict["creds"] = cred_task.id was_task = was_vulns_task.delay(data.org_acronym) - tasks_dict['was'] = was_task.id - #TODO: add task for XPANSE data + tasks_dict["was"] = was_task.id + # TODO: add task for XPANSE data # Return the new task id w/ "Processing" status return {"tasks_dict": tasks_dict, "status": "Processing"} @@ -6734,15 +6969,15 @@ def domain_permu_insert_dnstwist( LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- get_root_domains(), Issue 707 pe-reports/006 atc-framework --- # This function reuses the /rootdomains_by_org_uid endpoint - + # --- getDataSource(), Issue 708 pe-reports/007 atc-framework --- # This function reuses the /data_source_by_name endpoint - + # --- execute_hibp_breach_values(), Issue 709/008 atc-framework --- @api_router.put( @@ -6799,7 +7034,7 @@ def cred_breaches_hibp_insert( ).update( modified_daate=row_dict["modified_date"], exposed_cred_count=row_dict["exposed_cred_count"], - password_included=row_dict["password_included"] + password_included=row_dict["password_included"], ) update_count += 1 return ( @@ -6812,7 +7047,7 @@ def cred_breaches_hibp_insert( LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- execute_hibp_emails_values(), Issue 710 pe-reports/009 atc-framework --- @api_router.put( @@ -6867,14 +7102,13 @@ def cred_exp_hibp_insert( create_cnt += 1 # Return success message return ( - str(create_cnt) - + " records created in the credential_exposures table" + str(create_cnt) + " records created in the credential_exposures table" ) except ObjectDoesNotExist: LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- get_breach_uids(), Issue 010 atc-framework --- @api_router.get( @@ -6895,7 +7129,7 @@ def breach_uids(tokens: dict = Depends(get_api_key)): # If API key valid, make query breach_uids_data = list( CredentialBreaches.objects.all().values( - "breach_name", + "breach_name", "credential_breaches_uid", ) ) @@ -6909,7 +7143,7 @@ def breach_uids(tokens: dict = Depends(get_api_key)): LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- query_orgs(), Issue 011 atc-framework --- @api_router.get( @@ -6952,7 +7186,9 @@ def reported_orgs(tokens: dict = Depends(get_api_key)): response_model=List[schemas.SubdomainsByOrgUID], tags=["Retrieve subdomains for the specified org uid."], ) -def subdomains_by_org_uid(data: schemas.SubdomainsByOrgUIDInput, tokens: dict = Depends(get_api_key)): +def subdomains_by_org_uid( + data: schemas.SubdomainsByOrgUIDInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get subdomains for specified org uid.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") @@ -6961,11 +7197,8 @@ def subdomains_by_org_uid(data: schemas.SubdomainsByOrgUIDInput, tokens: dict = userapiTokenverify(theapiKey=tokens) # If API key valid, make query subdomains_by_org_uid_data = list( - RootDomains.objects.filter( - organizations_uid=data.org_uid - ).values( - "root_domains_uid__sub_domain", - "root_domain" + RootDomains.objects.filter(organizations_uid=data.org_uid).values( + "root_domains_uid__sub_domain", "root_domain" ) ) return subdomains_by_org_uid_data @@ -6973,7 +7206,7 @@ def subdomains_by_org_uid(data: schemas.SubdomainsByOrgUIDInput, tokens: dict = LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- insert_shodan_assets(), Issue 016 atc-framework --- @api_router.put( @@ -6991,48 +7224,104 @@ def shodan_assets_insert( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) - # If API key valid, insert intelx data - create_cnt = 0 - for row in data.exp_data: + # userapiTokenverify(theapiKey=tokens) + # If API key is valid, proceed with the operation + update_create_count = 0 + try: + mdl_data_source = MDL_DataSource.objects.get(name="Shodan") + + except MDL_DataSource.DoesNotExist: + LOGGER.warning("DataSource 'Shodan' not found.") + mdl_data_source = None # Set to None if DataSource is not found + + for row in data.asset_data: row_dict = row.__dict__ + try: - # Check if record already exists - ShodanAssets.objects.get( - organizations_uid=row_dict["organizations_uid"], - ip=row_dict["ip"], - port=row_dict["port"], - protocol=row_dict["protocol"], - timestamp=row_dict["timestamp"], - ) - # If record already exists, do nothing - except CredentialExposures.DoesNotExist: - # If record doesn't exist yet, create one - curr_org_inst = Organizations.objects.get( + org_instance = Organizations.objects.get( organizations_uid=row_dict["organizations_uid"] ) - ShodanAssets.objects.create( - # Need to fill this out - # credential_exposures_uid=uuid.uuid1(), - email=row_dict["email"], - organizations_uid=curr_org_inst, - root_domain=row_dict["root_domain"], - sub_domain=row_dict["sub_domain"], - modified_date=row_dict["modified_date"], - breach_name=row_dict["breach_name"], - name=row_dict["name"], + + acronym = org_instance.cyhy_db_name + + mdl_org = MDL_Organization.objects.get(acronym=acronym) + + mdl_asset_fields = { + "asn": row_dict.get("asn"), + "domains": row_dict.get("domains", []), + "hostnames": row_dict.get("hostnames", []), + "isn": row_dict.get("isn"), + "organization_name": row_dict.get("organization"), + "product": row_dict.get("product"), + "tags": row_dict.get("tags", []), + "country_code": row_dict.get("country_code"), + "location": row_dict.get("location"), + "data_source": mdl_data_source, + } + + mdl_obj, created = MDL_ShodanAssets.objects.update_or_create( + organization=mdl_org, # Directly use organizations_uid + ip=row_dict["ip"], + port=row_dict["port"], + protocol=row_dict["protocol"], + timestamp=timezone.make_aware( + parse_datetime(row_dict["timestamp"]), timezone.timezone.utc + ), + defaults=mdl_asset_fields, ) - create_cnt += 1 - # Return success message - return ( - str(create_cnt) - + " records created in the credential_exposures table" - ) + except Exception as e: + LOGGER.warning(f"Shodan Asset failed to save to MDL: {e}") + + try: + # Prepare the fields that will be used for create or update (only non-unique fields here) + asset_fields = { + "asn": row_dict.get("asn"), + "domains": row_dict.get("domains", []), + "hostnames": row_dict.get("hostnames", []), + "isn": row_dict.get("isn"), + "organization": row_dict.get("organization"), + "product": row_dict.get("product"), + "tags": row_dict.get("tags", []), + "country_code": row_dict.get("country_code"), + "location": row_dict.get("location"), + "data_source_uid_id": row_dict.get("data_source_uid"), + } + + # Use 'update_or_create' to either create or update the record + obj, created = ShodanAssets.objects.update_or_create( + organizations_uid=org_instance, # Directly use organizations_uid + ip=row_dict["ip"], + port=row_dict["port"], + protocol=row_dict["protocol"], + timestamp=timezone.make_aware( + datetime.strptime( + row_dict["timestamp"], "%Y-%m-%dT%H:%M:%S.%f" + ), + timezone.timezone.utc, + ), + defaults=asset_fields, + ) + + if created: + update_create_count += 1 + except Exception as e: + LOGGER.warning(f"Shodan Asset failed to save to PE DB: {e}") + continue + + # Return the success message with the count of created/updated records + return { + "message": f"{update_create_count} records created/updated in the shodan_assets table." + } + except ObjectDoesNotExist: - LOGGER.info("API key expired please try again") + LOGGER.info("API key expired or invalid. Please try again.") + return {"message": "Invalid API key or expired. Please try again."} + except Exception as e: + LOGGER.error(f"Error: {str(e)}") + return {"message": "An error occurred while processing the request."} else: - return {"message": "No api key was submitted"} - + return {"message": "No API key was submitted."} + # --- insert_shodan_vulns(), Issue 017 atc-framework --- @api_router.put( @@ -7050,51 +7339,136 @@ def shodan_vulns_insert( LOGGER.info(f"The api key submitted {tokens}") if tokens: try: - userapiTokenverify(theapiKey=tokens) + # userapiTokenverify(theapiKey=tokens) # If API key valid, insert intelx data create_cnt = 0 - for row in data.exp_data: + try: + mdl_data_source = MDL_DataSource.objects.get(name="Shodan") + except DataSource.DoesNotExist: + LOGGER.warning("DataSource for 'Shodan' not found.") + mdl_data_source = None # Set to None if DataSource is not found + for row in data.vuln_data: row_dict = row.__dict__ try: - CredentialExposures.objects.get( - breach_name=row_dict["breach_name"], - email=row_dict["email"], - ) - # If record already exists, do nothing - except CredentialExposures.DoesNotExist: - # If record doesn't exist yet, create one - curr_org_inst = Organizations.objects.get( + org_instance = Organizations.objects.get( organizations_uid=row_dict["organizations_uid"] ) - curr_source_inst = DataSource.objects.get( - data_source_uid=row_dict["data_source_uid"] - ) - curr_breach_inst = CredentialBreaches.objects.get( - breach_name=row_dict["breach_name"], + acronym = org_instance.cyhy_db_name + + mdl_org = MDL_Organization.objects.get(acronym=acronym) + + mdl_vuln_data = { + "organization_name": row_dict.get("organization"), + "cve": row_dict.get("cve"), + "severity": row_dict.get("severity"), + "cvss": row_dict.get("cvss"), + "summary": row_dict.get("summary"), + "product": row_dict.get("product"), + "attack_vector": row_dict.get("attack_vector"), + "av_description": row_dict.get("av_description"), + "attack_complexity": row_dict.get("attack_complexity"), + "ac_description": row_dict.get("ac_description"), + "confidentiality_impact": row_dict.get( + "confidentiality_impact" + ), + "ci_description": row_dict.get("ci_description"), + "integrity_impact": row_dict.get("integrity_impact"), + "ii_description": row_dict.get("ii_description"), + "availability_impact": row_dict.get("availability_impact"), + "ai_description": row_dict.get("ai_description"), + "tags": row_dict.get("tags"), + "domains": row_dict.get("domains"), + "hostnames": row_dict.get("hostnames"), + "isn": row_dict.get("isn"), + "asn": row_dict.get("asn"), + "data_source": mdl_data_source, + "type": row_dict.get("type"), + "name": row_dict.get("name"), + "potential_vulns": row_dict.get("potential_vulns"), + "mitigation": row_dict.get("mitigation"), + "server": row_dict.get("server"), + "is_verified": row_dict.get("is_verified"), + "banner": row_dict.get("banner"), + "version": row_dict.get("version"), + "cpe": row_dict.get("cpe"), + } + + mdl_obj, created = MDL_ShodanVulns.objects.update_or_create( + organization=mdl_org, # Directly use organizations_uid + ip=row_dict["ip"], + port=row_dict["port"], + protocol=row_dict["protocol"], + timestamp=timezone.make_aware( + parse_datetime(row_dict["timestamp"]) + ), + defaults=mdl_vuln_data, ) - CredentialExposures.objects.create( - # credential_exposures_uid=uuid.uuid1(), - email=row_dict["email"], - organizations_uid=curr_org_inst, - root_domain=row_dict["root_domain"], - sub_domain=row_dict["sub_domain"], - modified_date=row_dict["modified_date"], - breach_name=row_dict["breach_name"], - credential_breaches_uid=curr_breach_inst, - data_source_uid=curr_source_inst, - name=row_dict["name"], + + except Exception as e: + LOGGER.warning(f"Shodan Vuln failed to save to MDL: {e}") + + try: + vuln_data = { + "organization": row_dict.get("organization"), + "cve": row_dict.get("cve"), + "severity": row_dict.get("severity"), + "cvss": row_dict.get("cvss"), + "summary": row_dict.get("summary"), + "product": row_dict.get("product"), + "attack_vector": row_dict.get("attack_vector"), + "av_description": row_dict.get("av_description"), + "attack_complexity": row_dict.get("attack_complexity"), + "ac_description": row_dict.get("ac_description"), + "confidentiality_impact": row_dict.get( + "confidentiality_impact" + ), + "ci_description": row_dict.get("ci_description"), + "integrity_impact": row_dict.get("integrity_impact"), + "ii_description": row_dict.get("ii_description"), + "availability_impact": row_dict.get("availability_impact"), + "ai_description": row_dict.get("ai_description"), + "tags": row_dict.get("tags"), + "domains": row_dict.get("domains"), + "hostnames": row_dict.get("hostnames"), + "isn": row_dict.get("isn"), + "asn": row_dict.get("asn"), + "data_source_uid_id": row_dict.get("data_source_uid"), + "type": row_dict.get("type"), + "name": row_dict.get("name"), + "potential_vulns": row_dict.get("potential_vulns"), + "mitigation": row_dict.get("mitigation"), + "server": row_dict.get("server"), + "is_verified": row_dict.get("is_verified"), + "banner": row_dict.get("banner"), + "version": row_dict.get("version"), + "cpe": row_dict.get("cpe"), + } + + obj, created = ShodanVulns.objects.update_or_create( + organizations_uid=org_instance, # Directly use organizations_uid + ip=row_dict["ip"], + port=row_dict["port"], + protocol=row_dict["protocol"], + timestamp=timezone.make_aware( + datetime.strptime( + row_dict["timestamp"], "%Y-%m-%dT%H:%M:%S.%f" + ) + ), + defaults=vuln_data, ) - create_cnt += 1 + LOGGER.info(f"pe vuln created: {created}") + if created: + create_cnt += 1 + except Exception as e: + LOGGER.warning(f"Shodan Vuln failed to save to PE DB: {e}") + continue # Return success message - return ( - str(create_cnt) - + " records created in the credential_exposures table" - ) + return str(create_cnt) + " records created in the shodan vulns table" except ObjectDoesNotExist: LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} - + # --- get_demo_orgs(), Issue 018 atc-framework --- @api_router.get( @@ -7127,6 +7501,7 @@ def organizations_demo(tokens: dict = Depends(get_api_key)): else: return {"message": "No api key was submitted"} + # --- Endpoint for Orgs with assets query (no view) --- @api_router.post( "/orgs_and_assets", @@ -7136,7 +7511,9 @@ def organizations_demo(tokens: dict = Depends(get_api_key)): response_model=schemas.OrgAssetTaskResp, tags=["Call API endpoint to get list all orgs and their linked assets."], ) -def orgs_and_assets(data: schemas.OrgsAssetsPagedInput, tokens: dict = Depends(get_api_key)): +def orgs_and_assets( + data: schemas.OrgsAssetsPagedInput, tokens: dict = Depends(get_api_key) +): """Call API endpoint to get list all orgs and their linked assets.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") @@ -7161,7 +7538,9 @@ def orgs_and_assets(data: schemas.OrgsAssetsPagedInput, tokens: dict = Depends(g response_model=schemas.OrgAssetTaskResp, tags=["Get task status for orgs and assets task."], ) -async def get_orgs_and_assets_task_status(task_id: str, tokens: dict = Depends(get_api_key)): +async def get_orgs_and_assets_task_status( + task_id: str, tokens: dict = Depends(get_api_key) +): """Get task status for orgs and assets task.""" # Check for API key LOGGER.info(f"The api key submitted {tokens}") @@ -7190,5 +7569,207 @@ async def get_orgs_and_assets_task_status(task_id: str, tokens: dict = Depends(g return {"task_id": task_id, "status": task.state} except ObjectDoesNotExist: LOGGER.info("API key expired please try again") + else: + return {"message": "No api key was submitted"} + + +@api_router.get( + "/query_shodan_ips/{org_uid}", + dependencies=[ + Depends(get_api_key) + ], # Depends(RateLimiter(times=200, seconds=60))], + # response_model=List[schemas.OrgsReportOnContacts], + tags=["Get all ips to run through Shodan."], +) +def query_shodan_ips(org_uid: str, tokens: dict = Depends(get_api_key)): + """Create API endpoint to get all ips to run through Shodan..""" + # Check for API key + LOGGER.info(f"The api key submitted {tokens}") + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + # If API key valid, make query + ips_from_cidrs = Ips.objects.filter( + origin_cidr__organizations_uid=org_uid, + origin_cidr__isnull=False, + shodan_results=True, + current=True, + ).values_list("ip", flat=True) + + ips_from_subs = Ips.objects.filter( + ipssubs__sub_domain_uid__root_domain_uid__organizations_uid=org_uid, # Correct relationship traversal + shodan_results=True, # 'shodan_results' is True + ipssubs__sub_domain_uid__current=True, # 'current' is True for subdomains + current=True, # 'current' is True for Ips + ).values_list("ip", flat=True) + + # Convert the QuerySet to sets + in_first = set(ips_from_cidrs) + in_second = set(ips_from_subs) + + # Find IPs that are in the second query but not in the first + in_second_but_not_in_first = in_second - in_first + + # Combine the results + ips = list(ips_from_cidrs) + list(in_second_but_not_in_first) + + return ips + except ObjectDoesNotExist: + LOGGER.info("API key expired please try again") + else: + return {"message": "No api key was submitted"} + + +@api_router.put( + "/was_finding_insert_or_update", + dependencies=[Depends(get_api_key)], + # response_model=Dict[schemas.PshttDataBase], + tags=["Update or insert WAS finding"], +) +# @transaction.atomic +def was_finding_insert_or_update( + # tag: str, + data: schemas.WasFindingInsert, + tokens: dict = Depends(get_api_key), +): + """Create API endpoint to create a record in database.""" + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + LOGGER.info(f"The api key submitted {tokens}") + defaults={ + 'finding_type': data.finding_type, + 'webapp_id': data.webapp_id, + 'webapp_url': data.webapp_url, + 'webapp_name': data.webapp_name, + 'was_org_id': data.was_org_id, + 'name': data.name, + 'owasp_category': data.owasp_category, + 'severity': data.severity, + 'times_detected': data.times_detected, + 'cvss_v3_attack_vector': data.cvss_v3_attack_vector, + 'base_score': data.base_score, + 'temporal_score': data.temporal_score, + 'fstatus': data.fstatus, + 'last_detected': data.last_detected, + 'first_detected': data.first_detected, + 'potential': data.potential, + 'cwe_list': data.cwe_list, + 'wasc_list': data.wasc_list, + 'last_tested': data.last_tested, + 'fixed_date': data.fixed_date, + 'is_ignored': data.is_ignored, + 'is_remediated': True if data.fstatus == "FIXED" else False, + 'url': data.url, + 'qid': data.qid, + 'response': data.response + } + ( + was_finding_object, + created, + ) = WasFindings.objects.update_or_create( + finding_uid= data.finding_uid, + defaults=defaults, + ) + try: + ( + mdl_was_finding_object, + mdl_created, + ) = MDL_WasFindings.objects.update_or_create( + finding_uid= data.finding_uid, + defaults=defaults, + ) + except Exception: + LOGGER.info(f'Failed to insert WAS finding to MDL: {data.finding_uid}') + + if created: + LOGGER.info( + "New WAS finding record created for %s", data.was_org_id + ) + return { + "message": "New WAS finding created.", + "was_finding_obj": was_finding_object, + } + return { + "message": "WAS finding updated.", + "was_finding_obj": was_finding_object, + } + except Exception as e: + LOGGER.warning(e) + print("failed to insert or update") + LOGGER.info("API key expired please try again") + else: + return {"message": "No api key was submitted"} + +@api_router.post( + "/was_report_insert_or_update", + dependencies=[Depends(get_api_key)], + # response_model=Dict[schemas.PshttDataBase], + tags=["Update or insert Was Report."], +) +def was_report_insert_or_update( + # tag: str, + data: schemas.WasReportInsert, + tokens: dict = Depends(get_api_key), +): + """Create API endpoint to create a record in database.""" + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + # LOGGER.info(f"The api key submitted {tokens}") + LOGGER.info(data) + serialized_data = data.dict() + LOGGER.info(serialized_data) + # Pass serialized data to Celery task + was_report_task = insert_was_report.delay(serialized_data) + # was_report_task = insert_was_report.delay(data) + was_report_task_id = was_report_task.id + # Return the new task id w/ "Processing" status + return {"task_id": was_report_task_id, "status": "Processing"} + except ObjectDoesNotExist: + LOGGER.info("API key expired please try again") + except Exception as e: + print('In View Basic Exception') + LOGGER.error(e) + + else: + return {"message": "No api key was submitted"} + +@api_router.get( + "/was_report_insert_or_update/task/{task_id}", + dependencies=[ + Depends(get_api_key) + ], # Depends(RateLimiter(times=200, seconds=60))], + # response_model=schemas.WasReportTaskResp, + tags=["Check task status WAS Report insert."], +) +async def was_report_insert_task_status(task_id: str, tokens: dict = Depends(get_api_key)): + """Check task status WAS Report insert""" + # Check for API key + LOGGER.info(f"The api key submitted {tokens}") + if tokens: + try: + # userapiTokenverify(theapiKey=tokens) + # Retrieve task status + task = insert_was_report.AsyncResult(task_id) + # Return appropriate message for status + if task.state == "SUCCESS": + return { + "task_id": task_id, + "status": "Completed", + "result": task.result, + } + elif task.state == "PENDING": + return {"task_id": task_id, "status": "Pending"} + elif task.state == "FAILURE": + return { + "task_id": task_id, + "status": "Failed", + "error": str(task.result), + } + else: + return {"task_id": task_id, "status": task.state} + except ObjectDoesNotExist: + LOGGER.info("API key expired please try again") else: return {"message": "No api key was submitted"} \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/__init__.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/admin.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/admin.py new file mode 100644 index 00000000..8c38f3f3 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/apps.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/apps.py new file mode 100644 index 00000000..781a3893 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class DmzMiniDlConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "dmz_mini_dl" diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/migrations/__init__.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py new file mode 100644 index 00000000..b671f91f --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/models.py @@ -0,0 +1,6402 @@ +"""Django ORM models.""" +# Standard Python Libraries +import uuid + +# Third-Party Libraries +# from django.contrib.auth.models import User as AuthUser +from django.contrib.postgres.fields import ArrayField +from django.db import models +from netfields import InetAddressField + +# , NetManager + +manage_db = True +app_label_name = "xfd_mini_dl" + + +class ApiKey(models.Model): + """The ApiKey model.""" + + id = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for an API key object." + ) + created_at = models.DateTimeField( + auto_now_add=True, + db_column="created_at", + help_text="Date and time the API key object was created.", + ) + updated_at = models.DateTimeField( + auto_now=True, + db_column="updated_at", + help_text="Date and time the API key object was last updated.", + ) + last_used = models.DateTimeField( + db_column="last_used", + blank=True, + null=True, + help_text="Last date and time the API key was used.", + ) + hashed_key = models.TextField( + db_column="hashed_key", help_text="Cryptographic hash of the API key" + ) + last_four = models.TextField( + db_column="last_four", help_text="Last for characters of the API key" + ) + user = models.ForeignKey( + "User", + models.CASCADE, + db_column="user_id", + blank=True, + null=True, + help_text="FK: foreign key relationship to the user who owns the API key.", + ) + + class Meta: + """Meta class for ApiKey.""" + + app_label = app_label_name + managed = manage_db + db_table = "api_key" + + +class Cpe(models.Model): + """The Cpe model.""" + + id = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for a CPE Product object." + ) + name = models.CharField(max_length=255, help_text="Name of the product.") + version = models.CharField(max_length=255, help_text="Version of the product.") + vendor = models.CharField( + max_length=255, help_text="Vendorr who created the product." + ) + last_seen_at = models.DateTimeField( + db_column="last_seen_at", help_text="Last datetime the CPE was seen." + ) + + class Meta: + """The Meta class for Cpe.""" + + app_label = app_label_name + db_table = "cpe" + managed = manage_db # This ensures Django does not manage the table + unique_together = (("name", "version", "vendor"),) # Unique constraint + + +class Cve(models.Model): + """The Cve model.""" + + id = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for a CVE object." + ) + name = models.CharField( + unique=True, blank=True, null=True, max_length=255, help_text="Name of the CVE." + ) + published_at = models.DateTimeField( + db_column="published_at", + blank=True, + null=True, + help_text="Date the CVE was published by NIST.", + ) + modified_at = models.DateTimeField( + db_column="modified_at", + blank=True, + null=True, + help_text="Datte the CVE was modified.", + ) + status = models.CharField( + blank=True, null=True, max_length=255, help_text="Status of the CVE." + ) + description = models.TextField( + blank=True, null=True, help_text="Description of the CVE." + ) + cvss_v2_source = models.CharField( + db_column="cvss_v2_source", + blank=True, + null=True, + max_length=255, + help_text="Organization or entity that assigned a CVSS v2 (Common Vulnerability Scoring System version 2) score to a particular vulnerability.", + ) + cvss_v2_type = models.CharField( + db_column="cvss_v2_type", + blank=True, + null=True, + max_length=255, + help_text="Type of CVVS v2 score. (Primary, Secondary)", + ) + cvss_v2_version = models.CharField( + db_column="cvss_v2_version", + blank=True, + null=True, + max_length=255, + help_text="Version of the CVSS v2 score.", + ) + cvss_v2_vector_string = models.CharField( + db_column="cvss_v2_vector_string", + blank=True, + null=True, + max_length=255, + help_text="Textual representation of the specific metrics used to calculate the CVSS v2 score.", + ) + cvss_v2_base_score = models.CharField( + db_column="cvss_v2_base_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the severity of the vulnerability.", + ) + cvss_v2_base_severity = models.CharField( + db_column="cvss_v2_base_severity", + blank=True, + null=True, + max_length=255, + help_text="Qualitative categorization of a vulnerability's Base Score that helps assess its overall risk level in a more human-readable way.", + ) + cvss_v2_exploitability_score = models.CharField( + db_column="cvss_v2_exploitability_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the exploitability of the vulnerability.", + ) + cvss_v2_impact_score = models.CharField( + db_column="cvss_v2_impact_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the potential impact of the vulnerability.", + ) + cvss_v3_source = models.CharField( + db_column="cvss_v3_source", + blank=True, + null=True, + max_length=255, + help_text="Organization or entity that has provided or published the CVSS v3 score for a given vulnerability.", + ) + cvss_v3_type = models.CharField( + db_column="cvss_v3_type", + blank=True, + null=True, + max_length=255, + help_text="Type of CVVS v3 score. (Primary, Secondary)", + ) + cvss_v3_version = models.CharField( + db_column="cvss_v3_version", + blank=True, + null=True, + max_length=255, + help_text="Version of the CVSS v3 score.", + ) + cvss_v3_vector_string = models.CharField( + db_column="cvss_v3_vector_string", + blank=True, + null=True, + max_length=255, + help_text="Textual representation of the specific metrics used to calculate the CVSS v3 score.", + ) + cvss_v3_base_score = models.CharField( + db_column="cvss_v3_base_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the severity of the vulnerability.", + ) + cvss_v3_base_severity = models.CharField( + db_column="cvss_v3_base_severity", + blank=True, + null=True, + max_length=255, + help_text="Qualitative categorization of a vulnerability's Base Score that helps assess its overall risk level in a more human-readable way.", + ) + cvss_v3_exploitability_score = models.CharField( + db_column="cvss_v3_exploitability_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the exploitability of the vulnerability.", + ) + cvss_v3_impact_score = models.CharField( + db_column="cvss_v3_impact_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the potential impact of the vulnerability.", + ) + cvss_v4_source = models.CharField( + db_column="cvss_v4_source", + blank=True, + null=True, + max_length=255, + help_text="Organization or entity that has provided or published the CVSS v4 score for a given vulnerability.", + ) + cvss_v4_type = models.CharField( + db_column="cvss_v4_type", + blank=True, + null=True, + max_length=255, + help_text="Type of CVVS v4 score. (Primary, Secondary)", + ) + cvss_v4_version = models.CharField( + db_column="cvss_v4_version", + blank=True, + null=True, + max_length=255, + help_text="Version of the CVSS v4 score.", + ) + cvss_v4_vector_string = models.CharField( + db_column="cvss_v4_vector_string", + blank=True, + null=True, + max_length=255, + help_text="Textual representation of the specific metrics used to calculate the CVSS v4 score.", + ) + cvss_v4_base_score = models.CharField( + db_column="cvss_v4_base_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the severity of the vulnerability.", + ) + cvss_v4_base_severity = models.CharField( + db_column="cvss_v4_base_severity", + blank=True, + null=True, + max_length=255, + help_text="Qualitative categorization of a vulnerability's Base Score that helps assess its overall risk level in a more human-readable way.", + ) + cvss_v4_exploitability_score = models.CharField( + db_column="cvss_v4_exploitability_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the exploitability of the vulnerability.", + ) + cvss_v4_impact_score = models.CharField( + db_column="cvss_v4_impact_score", + blank=True, + null=True, + max_length=255, + help_text="Numerical value that quantifies the potential impact of the vulnerability.", + ) + weaknesses = models.TextField( + blank=True, + null=True, + help_text="Weaknesses (CWE) associated with the vulnerability.", + ) + references = models.TextField( + blank=True, + null=True, + help_text="URLs to references associated with the vulnerability.", + ) + dve_score = models.DecimalField( + max_digits=1000, + decimal_places=1000, + blank=True, + null=True, + help_text="CyberSixGill's Dynamic Vulnerability Exploit (DVE) Score, this state-of-the-art machine learning model automatically predicts the probability of a CVE being exploited.", + ) + + cpes = models.ManyToManyField( + Cpe, + related_name="cves", + blank=True, + help_text="Many to many relationship to list of affected Products (CPE).", + ) + # tickets = models.ManyToManyField("Ticket", related_name='cves', blank=True) + # vuln_scans = models.ManyToManyField("VulnScan", related_name='cves', blank=True) + + class Meta: + """The Meta class for Cve.""" + + app_label = app_label_name + managed = manage_db + db_table = "cve" + + def save(self, *args, **kwargs): + """Format the model before saving.""" + self.name = self.name.lower() + self.reverseName = ".".join(reversed(self.name.split("."))) + super().save(*args, **kwargs) + + +class Notification(models.Model): + """The Notification model.""" + + id = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for a notification object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Datetime the notification object was created.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Datetime the notification object was last updated in the database.", + ) + start_datetime = models.DateTimeField( + db_column="start_datetime", + blank=True, + null=True, + help_text="Datetime the notification should start being displayed on the cyhy dashboard.", + ) + end_datetime = models.DateTimeField( + db_column="end_datetime", + blank=True, + null=True, + help_text="Datetime the notification should stop being displayed on the cyhy dashboard.", + ) + maintenance_type = models.CharField( + db_column="maintenance_type", + blank=True, + null=True, + max_length=255, + help_text="Type of maintenance being done. (Major, Minor)", + ) + status = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="Status of the notification. (Active, Inactive)", + ) + updated_by = models.CharField( + db_column="updated_by", + blank=True, + null=True, + max_length=255, + help_text="User who updated the notification", + ) + message = models.TextField( + blank=True, + null=True, + help_text="Message to be displayed on the cyhy dashboard.", + ) + + class Meta: + """The Meta class for Notification.""" + + app_label = app_label_name + managed = manage_db + db_table = "notification" + + +class Organization(models.Model): + """The Organization model.""" + + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for a stakeholder Organization." + ) + created_at = models.DateTimeField( + db_column="created_at", + auto_now_add=True, + help_text="Date and time the organization object was created in the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + auto_now=True, + help_text="Last date and time the organization object was updated.", + ) + acronym = models.CharField( + unique=True, + blank=True, + null=True, + max_length=255, + help_text="Short name used to identify the organization. This should match ServiceNow and the cyhy mongo database org id.", + ) + retired = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean field to flag organizations that have been retired a", + ) + name = models.CharField(max_length=255, help_text="Full name of the organization") + root_domains = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + db_column="root_domains", + help_text="List of root domains attributed to the organization", + ) + ip_blocks = models.TextField( + db_column="ip_blocks", + help_text="IP blocks attributed to or provided by a stakeholder.", + ) # This field type is a guess. + is_passive = models.BooleanField( + db_column="is_passive", + help_text="Boolean to flag if only passive data collection can be used on the stakeholder's assets.", + ) + pending_domains = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + db_column="pending_domains", + help_text="List of domains that have not yet been run through the setup/enumeration process.", + ) # This field type is a guess + date_pe_first_reported = models.DateTimeField( + blank=True, + null=True, + help_text="Date that PE first delivered reports to this stakeholder", + ) + country = models.TextField( + blank=True, + null=True, + help_text="Abbreviation of the country the organization is based in.", + ) + country_name = models.TextField( + blank=True, + null=True, + help_text="Full name of the country the organization is based in.", + ) + state = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="Abbreviation of the US state the organization is based in.", + ) + region_id = models.CharField( + db_column="region_id", + blank=True, + null=True, + max_length=255, + help_text="Region number that the organization is found in.", + ) + state_fips = models.IntegerField( + db_column="state_fips", + blank=True, + null=True, + help_text="Federal Information Processing Standards code for the US state where the organization is found.", + ) + state_name = models.CharField( + db_column="state_name", + blank=True, + null=True, + max_length=255, + help_text="Full name of the US state the organization is based in.", + ) + county = models.TextField( + blank=True, + null=True, + help_text="Full name of the county the organization is found in.", + ) + county_fips = models.IntegerField( + db_column="county_fips", + blank=True, + null=True, + help_text="Federal Information Processing Standards code for the US county where the organization is found.", + ) + type = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="The type of organization, brought over from legacy crossfeed, but not sure if currently used.", + ) + pe_report_on = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if the organization receives PE reports.", + ) + pe_premium = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization receives a premium PE report.", + ) + pe_demo = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is in demo status for PE. This means that scans are run for the organization, but reports are not delivered.", + ) + agency_type = models.TextField( + blank=True, + null=True, + help_text="Type of organization pulled from the Cyhy mongo database (Federal, State, Local, Private).", + ) + is_parent = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if an organization has children organizations associated with it.", + ) + pe_run_scans = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean field to determine if pe scans should be run an organization's assets.", + ) + stakeholder = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is a cyhy stakeholder.", + ) + election = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if the organization is an election entity.", + ) + was_stakeholder = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is a WAS customer.", + ) + vs_stakeholder = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is a VS customer.", + ) + pe_stakeholder = models.BooleanField( + default=False, + null=True, + blank=True, + help_text="Boolean to flag if an organization is a PE customer.", + ) + receives_cyhy_report = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if the organization receives a cyhy report.", + ) + receives_bod_report = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if the organization receives a cyhy bod report.", + ) + receives_cybex_report = models.BooleanField( + blank=True, + null=True, + help_text="Boolean to flag if the organization receives a cyhy cybex report.", + ) + init_stage = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="First scan run in the VS scan process.", + ) + scheduler = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Type of scheduler the VS scan uses when running VS scans.", + ) + enrolled_in_vs_timestamp = models.DateTimeField( + db_column="enrolled_in_vs_timestamp", + auto_now=True, + help_text="Date the stakeholder enrolled in VS.", + ) + period_start_vs_timestamp = models.DateTimeField( + db_column="period_start_vs_timestamp", + auto_now=True, + help_text="Period start for the last report period VS ran.?????", + ) + report_types = models.JSONField( + null=True, + blank=True, + default=list, + help_text="List of types of CyHy reports the stakeholder receives ", + ) + scan_types = models.JSONField( + null=True, blank=True, default=list, help_text="Types of scans run by Cyhy." + ) + scan_windows = models.JSONField( + null=True, + blank=True, + default=list, + help_text="List of time windows when VS can scan a stakeholder's assets.", + ) + scan_limits = models.JSONField( + null=True, + blank=True, + default=list, + help_text="Limits placed on a VS scan by the stakeholder.", + ) + password = models.TextField( + blank=True, + null=True, + help_text="Encrypted password used to encrypt and decrypt reports sent to the stakeholder.", + ) + cyhy_period_start = models.DateField( + blank=True, + null=True, + help_text="Timestamp when scanning can begin for this organization.", + ) + location = models.ForeignKey( + "Location", + related_name="organizations", + on_delete=models.SET_NULL, + null=True, + blank=True, + help_text="Foreign Key linking to a related Location object.", + ) + # sectors = models.ManyToManyField("Sector", related_name='organizations', blank=True) covered in sectors table already + # cidrs = models.ManyToManyField("Cidr", related_name='organizations', blank=True) covered in the cidr table already + # vuln_scans = models.ManyToManyField("VulnScan", related_name='organizations', blank=True) + # hosts = models.ManyToManyField("Host", related_name='organizations', blank=True) covered in hosts table already + # port_scans = models.ManyToManyField("PortScan", related_name='organizations', blank=True) + parent = models.ForeignKey( + "self", + models.DO_NOTHING, + db_column="parent_id", + blank=True, + null=True, + help_text="Foreign Key linking to a related organization parent object.", + ) + created_by = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="created_by_id", + blank=True, + null=True, + help_text="Foreign Key linking to the user who create the organization.", + ) + org_type = models.ForeignKey( + "OrgType", + on_delete=models.CASCADE, + db_column="org_type_id", + blank=True, + null=True, + help_text="Foreign Key to the related orgType object. ", + ) + + class Meta: + """The meta class for Organization.""" + + app_label = app_label_name + managed = manage_db + db_table = "organization" + + +class OrganizationTag(models.Model): + """The OrganizationTag model.""" + + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for an Organization tag object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the organization tag was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the organization tag object was updated in the database.", + ) + name = models.CharField( + unique=True, + max_length=255, + help_text="The name of the tag used to link common organizations.", + ) + organization = models.ManyToManyField( + "Organization", + related_name="organization_tags", + blank=True, + help_text="Many to many relationship to link a tag to many organizations", + ) + + class Meta: + """The Meta class for OrganizationTag.""" + + app_label = app_label_name + managed = manage_db + db_table = "organization_tag" + + +# Probably can be removed and merged with a many to many relationship +# class OrganizationTagOrganizationsOrganization(models.Model): +# """The OrganizationTagOrganizationsOrganization model.""" + +# organization_tag_id = models.OneToOneField( +# OrganizationTag, +# models.DO_NOTHING, +# db_column="organizationTagId", +# primary_key=True, +# ) # The composite primary key (organizationTagId, organizationId) found, that is not supported. The first column is selected. +# organization_id = models.ForeignKey( +# Organization, models.DO_NOTHING, db_column="organizationId" +# ) + +# class Meta: +# """The Meta class for OrganizationTagOrganizationsOrganization.""" + +# managed = False +# db_table = "organization_tag_organizations_organization" +# unique_together = (("organizationTagId", "organizationId"),) + + +class QueryResultCache(models.Model): + """The QueryResultCache model.""" + + id = models.UUIDField( + primary_key=True, + help_text="Unique identifier for the query result object being cached.", + ) + identifier = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="Another identifier for the query being cached.", + ) + time = models.BigIntegerField( + help_text="Time the query was run against the database." + ) + duration = models.IntegerField(help_text="How long the query took to run.") + query = models.TextField( + help_text="The Query run against the database to be cached." + ) + result = models.TextField(help_text="Result from the query performed in Crossfeed.") + + class Meta: + """The Meta class for QueryResultCache.""" + + app_label = app_label_name + managed = manage_db + db_table = "query-result-cache" + + +class Role(models.Model): + """The Role model.""" + + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for the role object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the role object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the role object was updated in the database.", + ) + role = models.CharField( + max_length=255, + help_text="A role that a user can be assigned to, granting them specific access on the crossfeed platform.", + ) + approved = models.BooleanField( + help_text="A boolean flag to determine if the user has been approved to have the assigned role." + ) + created_by = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="created_by_id", + blank=True, + null=True, + help_text="Foreign key linking to the user who created the role in the database.", + ) + approved_by = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="approved_by_id", + related_name="role_approved_by_id_set", + blank=True, + null=True, + help_text="Foreign key to the user who approved the role assignation.", + ) + user = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="user_id", + related_name="role_user_id_set", + blank=True, + null=True, + help_text="Foreign key to the user being assigned the role.", + ) + organization = models.ForeignKey( + Organization, + models.DO_NOTHING, + db_column="organization_id", + blank=True, + null=True, + help_text="Foreign key to the organization the user is aligned to and whos data the user can access via their role.", + ) + + class Meta: + """The Meta class for Role.""" + + app_label = app_label_name + managed = manage_db + db_table = "role" + unique_together = (("user_id", "organization_id"),) + + +class SavedSearch(models.Model): + """The SavedSearch model.""" + + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for the Saved Search object" + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the saved search object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the saved search object was updated in the database.", + ) + name = models.CharField( + max_length=255, + help_text="User provided name of the saved search provided in the cyhy dashboard.", + ) + search_term = models.CharField( + db_column="search_term", + max_length=255, + help_text="The term being searched for in the cyhy dashboard.", + ) + sort_direction = models.CharField( + db_column="sort_direction", + max_length=255, + help_text="Direction of the sort (asc or desc).", + ) + sort_field = models.CharField( + db_column="sort_field", max_length=255, help_text="The field to sort based on." + ) + count = models.IntegerField( + help_text="Number of results returned when the search was run." + ) + filters = models.JSONField(help_text="Filters applied in the search.") + search_path = models.CharField( + db_column="search_path", + max_length=255, + help_text="Search path used to call create the search against the ORM.", + ) + # create_vulnerabilities = models.BooleanField(db_column="create_vulnerabilities", help_text="") # No longer used + # vulnerability_template = models.JSONField(db_column="vulnerability_template", help_text="") # No longer used + created_by = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="created_by_id", + blank=True, + null=True, + help_text="Foreign key linking to the user who created the saved search in the dashboard.", + ) + + class Meta: + """The Meta class for SavedSearch.""" + + app_label = app_label_name + managed = manage_db + db_table = "saved_search" + + +class Scan(models.Model): + """The Scan model.""" + + id = models.UUIDField( + primary_key=True, + help_text="Unique identifier for a cyhy dashboard scan object.", + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the scan object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the scan object was updated in the database.", + ) + name = models.CharField( + max_length=255, help_text="The name of the cyhy dashboard scan." + ) + arguments = models.JSONField( + help_text="A dictionary of arguments to pass to the scan." + ) + frequency = models.IntegerField( + help_text="How often the scan should run in seconds." + ) + last_run = models.DateTimeField( + db_column="last_run", + blank=True, + null=True, + help_text="Last day the scan was run.", + ) + is_granular = models.BooleanField( + db_column="is_granular", + help_text="A boolean flag to specify if the scan is granular. Granular scans are only run on specified organizations. Global scans cannot be granular scans.", + ) + is_user_modifiable = models.BooleanField( + db_column="is_user_modifiable", + blank=True, + null=True, + help_text="Whether the scan is user-modifiable. User-modifiable scans are granular scans that can be viewed and toggled on/off by organization admins themselves.", + ) + is_single_scan = models.BooleanField( + db_column="is_single_scan", + help_text="A boolean to flag scans that should only be run once and not on a reoccuring basis.", + ) + manual_run_pending = models.BooleanField( + db_column="manual_run_pending", + help_text="A boolean to flag if a manually called scan is still waiting to be run.", + ) + created_by = models.ForeignKey( + "User", + models.DO_NOTHING, + db_column="created_by", + blank=True, + null=True, + help_text="A foreign key linking to the user who created the scan.", + ) + organizations = models.ManyToManyField( + Organization, + related_name="scans", + blank=True, + help_text="A many to many relationship linking to all the organizations the scan should be run on.", + ) + organization_tags = models.ManyToManyField( + OrganizationTag, + related_name="scans", + blank=True, + help_text="A many to many relationship linking to all the organization tags that should be run on.", + ) + + class Meta: + """The Meta class for Scan.""" + + app_label = app_label_name + managed = manage_db + db_table = "scan" + + +class ScanTask(models.Model): + """The ScanTask model.""" + + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for a scan task object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the scan task object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the scan task object was updated in the database.", + ) + status = models.TextField( + help_text="The scan task's status. ('created', 'queued', 'requested', 'started', 'failed','finished')" + ) + type = models.TextField(help_text="Type of scan task. ('fargate', 'lambda')") + fargate_task_arn = models.TextField( + db_column="fargate_task_arn", + blank=True, + null=True, + help_text="Unique identifier for the fargate container running the task.", + ) + input = models.TextField( + blank=True, + null=True, + help_text="All data necessary to run the scan task. (organizations, scan_id, scanName, scanTaskId, isSingleScan)", + ) + output = models.TextField( + blank=True, + null=True, + help_text="All the data returned from the scan task, dependant on the type of scan.", + ) + requested_at = models.DateTimeField( + db_column="requested_at", + blank=True, + null=True, + help_text="Date and time the scan task was requested.", + ) + started_at = models.DateTimeField( + db_column="started_at", + blank=True, + null=True, + help_text="Date and time the scan task was started.", + ) + finished_at = models.DateTimeField( + db_column="finished_at", + blank=True, + null=True, + help_text="Date and time the scan task finished.", + ) + queued_at = models.DateTimeField( + db_column="queued_at", + blank=True, + null=True, + help_text="Date and time the scan task was added to the queue.", + ) + organization = models.ForeignKey( + Organization, + models.DO_NOTHING, + db_column="organization_id", + blank=True, + null=True, + help_text="Foreign key to the organization instance the scan is being run on if it is a single scan.", + ) + scan = models.ForeignKey( + Scan, + models.DO_NOTHING, + db_column="scan_id", + blank=True, + null=True, + help_text="Foreign key to the scan the scan task was based off of.", + ) + organization_tags = models.ManyToManyField( + OrganizationTag, + related_name="scan_tasks", + blank=True, + help_text="List of organization tags that the scan task is running on.", + ) + + class Meta: + """The Meta class for ScanTask.""" + + app_label = app_label_name + managed = manage_db + db_table = "scan_task" + + +class Service(models.Model): + """The Service model.""" + + id = models.UUIDField( + primary_key=True, + help_text="Unique identifier for a web service running on a stakeholders attack surface.", + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the service object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the service object was updated in the database.", + ) + service_source = models.TextField( + db_column="service_source", + blank=True, + null=True, + help_text="The source of the service, which scan identified the service.", + ) + port = models.IntegerField(help_text="The port the service is running on.") + service = models.TextField(blank=True, null=True, help_text="Name of the service.") + last_seen = models.DateTimeField( + db_column="last_seen", + blank=True, + null=True, + help_text="Late date the service was seen running on the asset.", + ) + banner = models.TextField( + blank=True, + null=True, + help_text="Text that is automatically sent back to a client when they connect to the service.", + ) + products = models.JSONField(help_text="Products identified running on the port.") + censys_metadata = models.JSONField( + db_column="censys_metadata", + help_text="Metadata provided from the Censys scan of the service.", + ) + censys_ipv4_results = models.JSONField( + db_column="censys_ipv4_results", + help_text="IPv4 results provided from the Censys scan of the service.", + ) + intrigue_ident_results = models.JSONField( + db_column="intrigue_ident_results", + help_text="Additional details about the service provided by Intrigue scans.", + ) + shodan_results = models.JSONField( + db_column="shodan_results", + help_text="Details about the service identified through the Shodan scan.", + ) + wappalyzer_results = models.JSONField( + db_column="wappalyzer_results", + help_text="Details about the service identified by the wappalyzer scan.", + ) + domain = models.ForeignKey( + "SubDomains", + models.DO_NOTHING, + db_column="domain_id", + blank=True, + null=True, + help_text="Foreign key relationship to the domain the service is running on.", + ) + discovered_by = models.ForeignKey( + Scan, + models.DO_NOTHING, + db_column="discovered_by_id", + blank=True, + null=True, + help_text="Foreign key to the scan that discovered the service.", + ) + + class Meta: + """The Meta class for Service.""" + + app_label = app_label_name + managed = manage_db + db_table = "service" + unique_together = (("port", "domain"),) + + +class User(models.Model): + """The User model.""" + + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for a user object." + ) + cognito_id = models.CharField( + db_column="cognitoId", + unique=True, + blank=True, + null=True, + max_length=255, + help_text="Identifier for the user in the cognito system. This is necessary to log into the cyhy dashboard application.", + ) + login_gov_id = models.CharField( + db_column="login_gov_id", + unique=True, + blank=True, + null=True, + max_length=255, + help_text="Identifier for the user in the login.gov system. This is also used to log in to the cyhy dashboard.", + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the user object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the user object was updated in the database.", + ) + first_name = models.CharField( + db_column="first_name", max_length=255, help_text="First name of the user." + ) + last_name = models.CharField( + db_column="last_name", max_length=255, help_text="Last name of the user." + ) + full_name = models.CharField( + db_column="full_name", max_length=255, help_text="Full name of the user." + ) + email = models.CharField( + unique=True, max_length=255, help_text="User's email address." + ) + invite_pending = models.BooleanField( + db_column="invite_pending", + help_text="A boolean field flagging if the user's invite is pending.", + ) + login_blocked_by_maintenance = models.BooleanField( + db_column="login_blocked_by_maintenance", + help_text="A boolean flag identifying whether the user is blocked by maintenance to login", + ) + date_accepted_terms = models.DateTimeField( + db_column="date_accepted_terms", + blank=True, + null=True, + help_text="Date the user accepted the cyhy dashboard terms of service.", + ) + accepted_terms_version = models.TextField( + db_column="accepted_terms_version", + blank=True, + null=True, + help_text="The version of the the terms of service the user accepted.", + ) + last_logged_in = models.DateTimeField( + db_column="last_logged_in", + blank=True, + null=True, + help_text="Datetime the last time the user logged in.", + ) + user_type = models.TextField( + db_column="user_type", + help_text="The type of user. This determines what parts of the cyhy dashboard can view and what data he is permitted to see.", + ) + region_id = models.CharField( + db_column="region_id", + blank=True, + null=True, + max_length=255, + help_text="What region the user belongs to.", + ) + state = models.CharField( + blank=True, + null=True, + max_length=255, + help_text="The state the user resides in.", + ) + okta_id = models.CharField( + db_column="okta_id", + unique=True, + blank=True, + null=True, + max_length=255, + help_text="The Okta id associated with the user.", + ) + + class Meta: + """The Meta class for User.""" + + app_label = app_label_name + managed = manage_db + db_table = "user" + + +class Vulnerability(models.Model): + """The Vulnerability model.""" + + id = models.UUIDField( + primary_key=True, + help_text="Unique identifier for a vulnerability object found in the cyhy dashboard", + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the vulnerability object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the vulnerability object was updated in the database.", + ) + last_seen = models.DateTimeField( + db_column="last_seen", + blank=True, + null=True, + help_text="Last date the vulnerability was seen.", + ) + title = models.TextField(help_text="The name or title of the vulnerability.") + cve = models.TextField( + blank=True, + null=True, + help_text="CVE (Common Vulnerabilities and Exposures) id for the vulnerability.", + ) + cwe = models.TextField( + blank=True, + null=True, + help_text="Common Weakness Enumeration (CWE) id for the weakness or vulnerability.", + ) + cpe = models.TextField( + blank=True, + null=True, + help_text="Common Platform Enumeration (CPE) id for the product the vulnerability was found on.", + ) + description = models.TextField( + help_text="Human readable description of the vulnerability if available." + ) + references = models.JSONField( + help_text="Additional links to references and sources associates with the vulnerability." + ) + cvss = models.DecimalField( + max_digits=100, + decimal_places=5, + blank=True, + null=True, + help_text="CVSS (Common Vulnerability Scoring System) is the score reperesenting the severity of the vulnerability from 0 (None) to 10 (Critical)", + ) + severity = models.TextField( + blank=True, + null=True, + help_text="The severity level of the vulnerability determined by the cvss score. (None, Low, Medium, High, Critical)", + ) + needs_population = models.BooleanField( + db_column="needs_population", + help_text="A boolean field to flag vulnerabilities that need to be populated additional findings.", + ) + state = models.TextField( + help_text="The state the vulnerability is in, as of the last scan (Open, Closed)" + ) + substate = models.TextField( + help_text="Substate of the vulnerability ('unconfirmed', 'exploitable', 'false-positive', 'accepted-risk', 'remediated')" + ) + source = models.TextField(help_text="The scan that identified the vulnerability.") + notes = models.TextField( + help_text="Notes about the vulnerability, provided by the user of the cyhy dashboard." + ) + actions = models.JSONField( + help_text="A list of state changes of the vulnerability, tracking its status from intially created to closed." + ) + structured_data = models.JSONField( + db_column="structured_data", + help_text="Any additional data that does not fit into the vulnerability table pertinent to the end user.", + ) + is_kev = models.BooleanField( + db_column="is_kev", + blank=True, + null=True, + help_text="A boolean field to flag if a vulnerability has been on the CISA Known Exploited Vulnerability (KEV) list.", + ) + kev_results = models.JSONField( + db_column="kev_results", + blank=True, + null=True, + help_text="The CISA provided KEV information assocaited with KEV vulnerabilities.", + ) + domain = models.ForeignKey( + "SubDomains", + models.DO_NOTHING, + db_column="domain_id", + blank=True, + null=True, + help_text="Foreign key relationship to the domain the vulnerability was found on.", + ) + service = models.ForeignKey( + Service, + models.DO_NOTHING, + db_column="service_id", + blank=True, + null=True, + help_text="Foreign key relationship to the service the vulnerability was found on.", + ) + + class Meta: + """The Meta class for Vulnerability.""" + + app_label = app_label_name + managed = manage_db + db_table = "vulnerability" + unique_together = (("domain", "title"),) + + +class Webpage(models.Model): + """The Webpage model.""" + + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for the webpage object." + ) + created_at = models.DateTimeField( + db_column="created_at", + help_text="Date the webpage object was added to the database.", + ) + updated_at = models.DateTimeField( + db_column="updated_at", + help_text="Last date the webpage object was updated in the database.", + ) + synced_at = models.DateTimeField( + db_column="synced_at", + blank=True, + null=True, + help_text="When this model was last synced with Elasticsearch.", + ) + last_seen = models.DateTimeField( + db_column="last_seen", + blank=True, + null=True, + help_text="Last time the webpage was seen.", + ) + s3key = models.TextField( + db_column="s3Key", + blank=True, + null=True, + help_text="The AWS S3 key that corresponds to this webpage's contents.", + ) + url = models.TextField(help_text="URL to the webpage.") + status = models.DecimalField( + max_digits=100, decimal_places=5, help_text="The status of the HTTP response." + ) + response_size = models.DecimalField( + db_column="response_size", + max_digits=100, + decimal_places=5, + blank=True, + null=True, + help_text="The size of the url response.", + ) + headers = models.JSONField(help_text="The header returned from the url response.") + domain = models.ForeignKey( + "SubDomains", + models.DO_NOTHING, + db_column="domain_id", + blank=True, + null=True, + help_text="The domain associated with the webpage.", + ) + discovered_by = models.ForeignKey( + Scan, + models.DO_NOTHING, + db_column="discovered_by_id", + blank=True, + null=True, + help_text="The scan that discovered the webpage.", + ) + + class Meta: + """The Meta class for Webpage.""" + + app_label = app_label_name + managed = manage_db + db_table = "webpage" + unique_together = (("url", "domain"),) + + +# ######## VS Models ######### +class TicketEvent(models.Model): + """The TicketEvent model.""" + + id = models.UUIDField( + primary_key=True, + editable=False, + help_text="Unique id for a ticket event object in the database.", + ) + reference = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="The identifier for the vulnerability scan related to the event", + ) + vuln_scan = models.ForeignKey( + "VulnScan", + on_delete=models.CASCADE, + db_column="vuln_scan_id", + null=True, + blank=True, + related_name="ticket_events", + help_text="A foreign key relationship to the Vuln scan related to the event.", + ) + action = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Event action type. (OPENED, VERIFIED, CHANGED, CLOSED, REOPENED, UNVERIFIED)", + ) + reason = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Short description of the event", + ) + event_timestamp = models.DateTimeField( + null=True, blank=True, help_text="Timestamp indicating when the event occurred" + ) + delta = models.JSONField( + default=list, help_text="List of what changed; only applies to 'CHANGED' events" + ) + ticket = models.ForeignKey( + "Ticket", + on_delete=models.CASCADE, + db_column="ticket_id", + null=True, + blank=True, + related_name="ticket_events", + help_text="Foreign key relationship to the ticket the event references.", + ) + + class Meta: + """The Meta class for TicketEvent.""" + + app_label = app_label_name + managed = manage_db + db_table = "ticket_event" + unique_together = ("event_timestamp", "ticket", "action") + + +class VulnScan(models.Model): + """The VS Vuln Scan model.""" + + id = models.CharField( + max_length=255, + primary_key=True, + help_text="Unique identifier for the webpage object.", + ) + cert_id = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Id to look up a vulnerability int the CERT Vulnerability Notes Database. https://www.kb.cert.org/vuls/", + ) + cpe = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Common Platform Enumeration (CPE) id for the product the vulnerability was found on.", + ) + cve_string = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="CVE (Common Vulnerabilities and Exposures) id for the vulnerability.", + ) + cve = models.ForeignKey( + Cve, + related_name="vuln_scans", + on_delete=models.CASCADE, + blank=True, + null=True, + help_text="Foreign key relationship to the related CVE object.", + ) + cvss_base_score = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Numerical value that measures the severity of a vulnerability using the Common Vulnerability Scoring System (CVSS)", + ) + cvss_temporal_score = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Score representing a vulnerabilities urgency at specific points in time.", + ) + cvss_temporal_vector = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A textual representation of the metric values used to determine the temporal score.", + ) + cvss_vector = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A textual representation of the set of CVSS metrics.", + ) + description = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Description of the vulnerability, according to the vulnerability scanner.", + ) + exploit_available = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A boolean field flagging whether or not an exploit is available, according to the vulnerability scanner.", + ) + exploitability_ease = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Ease of exploitation, according to the vulnerability scanner.", + ) + ip_string = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="IPv4 or IPv6 address where the vulnerability was identified.", + ) + ip = models.ForeignKey( + "Ip", + related_name="vuln_scans", + on_delete=models.CASCADE, + blank=True, + null=True, + help_text="Foreign key relationship to the related IP object.", + ) + latest = models.BooleanField( + default=False, + help_text="A boolean field flagging if this is the latest vulnerability scan of this port/protocol/host.", + ) + owner = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Acronym of the organization that claims the IP address associated with this vulnerability scan.", + ) + osvdb_id = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Open Source Vulnerability Database identifier for the detected vulnerability.", + ) + organization = models.ForeignKey( + Organization, + related_name="vuln_scans", + on_delete=models.CASCADE, + blank=True, + null=True, + help_text="Foreign key relationship linking to the related Organization object.", + ) + patch_publication_timestamp = models.DateTimeField( + blank=True, + null=True, + help_text="Date when a patch was published for this vulnerability", + ) + cisa_known_exploited = models.DateTimeField(blank=True, null=True, help_text="????") + port = models.IntegerField( + blank=True, + null=True, + help_text="Number of the port that was vulnerability scanned", + ) + port_protocol = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Protocol for the vulnerable port in this scan ('tcp' or 'udp')", + ) + risk_factor = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Risk factor of the detected vulnerability according to the vulnerability scanner", + ) + script_version = models.CharField( + max_length=255, blank=True, null=True, help_text="Script version string" + ) + see_also = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Additional reference(s) for this vulnerability provided by the vulnerability scanner", + ) + service = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Service detected at the vulnerable port in this scan", + ) + severity = models.IntegerField( + blank=True, + null=True, + help_text="CVSS v2.0 severity rating from the vulnerability scanner.", + ) + solution = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Solution to mitigate the detected vulnerability, according to the vulnerability scanner", + ) + source = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Source of the vulnerability scan (e.g. 'nessus').", + ) + synopsis = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Brief overview of the vulnerability.", + ) + vuln_detection_timestamp = models.DateTimeField( + blank=True, + null=True, + help_text="Timestamp indicating when the vulnerability was detected.", + ) + vuln_publication_timestamp = models.DateTimeField( + blank=True, null=True, help_text="Vulnerability publication date." + ) + xref = models.CharField( + max_length=255, blank=True, null=True, help_text="External reference." + ) + cwe = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Common Weakness Enumeration (CWE) id for the weakness or vulnerability.", + ) + bid = models.CharField( + max_length=255, blank=True, null=True, help_text="Bugtraq ID" + ) + exploited_by_malware = models.BooleanField( + default=False, + help_text="A boolean field to flag if the vuln type has been exploited by a known malware.", + ) + thorough_tests = models.BooleanField( + default=False, + help_text="Boolean field to flag if more thorough tests have been run on the vulnerability for confirmation.", + ) + cvss_score_rationale = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Rationale for the cvss score given to the vulnerability.", + ) + cvss_score_source = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="The source that determined the cvss score for this vulnerability.", + ) + cvss3_base_score = models.DecimalField( + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="CVSS version 3 base score.", + ) + cvss3_vector = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A textual representation of the set of CVSS version 3 metrics.", + ) + cvss3_temporal_vector = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="A textual representation of the metric values used to determine the temporal score.", + ) + cvss3_temporal_score = models.DecimalField( + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="Score representing a vulnerabilities urgency at specific points in time.", + ) + asset_inventory = models.BooleanField(default=False, help_text="????") + plugin_id = models.CharField( + max_length=255, blank=True, null=True, help_text="ID of the plugin." + ) + plugin_modification_date = models.DateTimeField( + blank=True, + null=True, + help_text="Latest modification date of the vulnerability scanner plugin that detected this vulnerability.", + ) + plugin_publication_date = models.DateTimeField( + blank=True, + null=True, + help_text="Publication date of the vulnerability scanner plugin that detected this vulnerability.", + ) + plugin_name = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Name of the vulnerability scanner plugin that detected this vulnerability.", + ) + plugin_type = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Vulnerability scanner plugin type.", + ) + plugin_family = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Family of the plugin run by the vulnerability scanner that detected this vulnerability.", + ) + f_name = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Filename of the vulnerability scanner plugin that detected this vulnerability.", + ) + cisco_bug_id = models.CharField( + max_length=255, blank=True, null=True, help_text="??????" + ) + cisco_sa = models.CharField( + max_length=255, blank=True, null=True, help_text="??????" + ) + plugin_output = models.TextField( + blank=True, + null=True, + help_text="Plugin-specific output from the vulnerability scanner", + ) + # snapshots = models.ManyToManyField(Snapshot, related_name='vuln_scans') + # ticket_events = models.ManyToManyField(TicketEvent, related_name='vuln_scans') + other_findings = models.JSONField( + default=dict, + blank=True, + help_text="Additional data collected by the VS vuln scan that is not commonly seen.", + ) + + class Meta: + """The Meta class for VulnScan.""" + + app_label = app_label_name + managed = manage_db + db_table = "vuln_scan" + + +class Cidr(models.Model): + """The Cidr Model.""" + + id = models.UUIDField( + primary_key=True, + editable=False, + help_text="Unique idenifier for the Cidr object.", + ) + created_date = models.DateTimeField( + auto_now_add=True, help_text="Date the cidr object was added to the database." + ) + network = InetAddressField( + null=True, blank=True, unique=True, help_text="The cidr block" + ) # models.TextField() # This field type is a guess. + start_ip = InetAddressField( + null=True, blank=True, help_text="The first IP address in the cidr block." + ) + end_ip = InetAddressField( + null=True, blank=True, help_text="The last IP address in the cidr block." + ) + retired = models.BooleanField( + null=True, + blank=True, + help_text="A boolean field flagging if the cidr has been retired.", + ) + updated_at = models.DateTimeField( + auto_now=True, + help_text="The last time the cidr object was updated in the database.", + ) + insert_alert = models.TextField( + blank=True, + null=True, + help_text="An alert message specifying any conflicts when inserting the cidr into the database.", + ) + first_seen = models.DateField( + blank=True, null=True, help_text="First time the cidr was seen." + ) + last_seen = models.DateField( + blank=True, null=True, help_text="Last time the cidr was seen." + ) + current = models.BooleanField( + blank=True, + null=True, + help_text="A boolean field flagging if the cidr is current. If it is False it should not be run through any scans.", + ) + data_source = models.ForeignKey( + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + blank=True, + null=True, + help_text="Foreign key relationship to the data source that inserted the cidr object.", + ) + + organizations = models.ManyToManyField( + Organization, + related_name="cidrs", + blank=True, + help_text="Foreign key relationship to the organization that owns the cidr object.", + ) + + class Meta: + """The Meta class for Cidr.""" + + app_label = app_label_name + managed = manage_db + db_table = "cidr" + indexes = [models.Index(fields=["network"])] + + +class Location(models.Model): + """The Location model.""" + + id = models.UUIDField( + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text="Unique identifier for a location object.", + ) + name = models.CharField( + max_length=255, null=True, blank=True, help_text="Name of the location." + ) + country_abrv = models.CharField( + max_length=255, null=True, blank=True, help_text="Country abbreviation." + ) + country = models.CharField( + max_length=255, null=True, blank=True, help_text="Full name of the country." + ) + county = models.CharField( + max_length=255, null=True, blank=True, help_text="Name of the county." + ) + county_fips = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Federal Information Processing Standards code for the US county where the organization is found.", + ) + gnis_id = models.CharField( + max_length=255, + null=True, + blank=True, + unique=True, + help_text="(Geographic Names Information System ID) is a unique identifier assigned to geographic features in the GNIS database.", + ) + state_abrv = models.CharField( + max_length=255, null=True, blank=True, help_text="State abbreviation." + ) + state_fips = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Federal Information Processing Standards code for the US state where the organization is found.", + ) + state = models.CharField( + max_length=255, null=True, blank=True, help_text="Full name of the state." + ) + + class Meta: + """The Meta class for Location.""" + + app_label = app_label_name + managed = manage_db + db_table = "location" + indexes = [models.Index(fields=["gnis_id"])] + + +class Sector(models.Model): + """The Sector model.""" + + id = models.UUIDField( + primary_key=True, + editable=False, + default=uuid.uuid4, + help_text="Unique identifier for a sector object in the database.", + ) + name = models.CharField( + max_length=255, null=True, blank=True, help_text="The name of the sector." + ) + acronym = models.CharField( + max_length=255, + null=True, + blank=True, + unique=True, + help_text="The short name of the sector.", + ) + retired = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field flagging if the sector has been retired.", + ) + + organizations = models.ManyToManyField( + Organization, + related_name="sectors", + blank=True, + help_text="Many to many relationship between sectors and organizations.", + ) + + class Meta: + """The Meta class for Sector.""" + + app_label = app_label_name + managed = manage_db + db_table = "sector" + indexes = [models.Index(fields=["acronym"])] + + +class Host(models.Model): + """The Host model.""" + + id = models.CharField( + max_length=255, + primary_key=True, + help_text="Unique identifier for a host object in the database.", + ) + ip_string = models.CharField( + max_length=255, null=True, blank=True, help_text="The IP address of the host." + ) + ip = models.ForeignKey( + "Ip", + related_name="hosts", + on_delete=models.SET_NULL, + null=True, + blank=True, + help_text="Foreign key relationship to the related model.", + ) + updated_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp of the last time the host object was updated.", + ) + latest_netscan_1_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed the NETSCAN1.", + ) + latest_netscan_2_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed the NETSCAN2.", + ) + latest_vulnscan_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed the PORTSCAN.", + ) + latest_portscan_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed the VULNSCAN.", + ) + latest_scan_completion_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamps indicating last time host completed all scans.", + ) + location_longitude = models.DecimalField( + max_digits=10, + decimal_places=6, + null=True, + blank=True, + help_text="Longitude of host, according to geolocation database", + ) + location_latitude = models.DecimalField( + max_digits=10, + decimal_places=6, + null=True, + blank=True, + help_text="Latitude of host, according to geolocation database", + ) + priority = models.IntegerField( + null=True, + blank=True, + help_text="Scan priority of this host document, from -16 (most urgent) to 1 (least urgent)", + ) + next_scan_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp indicating when this host document is scheduled to be scanned next; a value of null indicates that the host document has a status other than 'DONE' (i.e. currently queued up for a scan or running a scan)", + ) + rand = models.DecimalField( + max_digits=10, + decimal_places=6, + null=True, + blank=True, + help_text="A random number between 0 and 1 used to randomize scan order", + ) + curr_stage = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Current scan stage for this host document", + ) + host_live = models.BooleanField( + null=True, + blank=True, + help_text="Whether or not a live host was detected at this host document’s IP address by the port scanner", + ) + host_live_reason = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Reason given by the port scanner as to whether or not this host document represents a live host", + ) + status = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Current scan status for this host document. (WAITING, READY, RUNNING, DONE)", + ) + organization = models.ForeignKey( + Organization, + related_name="hosts", + on_delete=models.CASCADE, + null=True, + blank=True, + help_text="Foreign key relationship to the organization that owns the host.", + ) + + class Meta: + """The Meta class for Host.""" + + app_label = app_label_name + managed = manage_db + db_table = "host" + indexes = [ + models.Index(fields=["ip_string"]), + ] + + +class Ip(models.Model): + """The Ip model.""" + + # id = models.UUIDField(primary_key=True, editable=False, default=uuid.uuid4) + ip_hash = models.TextField( + primary_key=True, help_text="A hash of the IP used as a unique identifier." + ) + organization = models.ForeignKey( + Organization, + related_name="ips", + on_delete=models.CASCADE, + help_text="Foreign key relationship to the organization that owns the IP.", + ) + created_timestamp = models.DateTimeField( + auto_now_add=True, + help_text="Timestamp the cidr object was added to the database.", + ) + updated_timestamp = models.DateTimeField( + null=True, + blank=True, + auto_now=True, + help_text="Timestamp of the last time the IP object was updated.", + ) + last_seen_timestamp = models.DateTimeField( + null=True, blank=True, help_text="Timestamp of the last time the IP was seen." + ) + ip = models.GenericIPAddressField( + null=True, blank=True, help_text="The IP address." + ) + live = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if the IP is live as of the last scan.", + ) + false_positive = models.BooleanField( + null=True, + blank=True, + help_text="A boolean field that marks if the IP was incorrectly attributed to the stakeholder.", + ) + from_cidr = models.BooleanField( + null=True, blank=True, help_text="The cidr block the IP originated from." + ) + retired = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if the IP is no longer owned by the organization.", + ) + last_reverse_lookup = models.DateTimeField( + blank=True, + null=True, + help_text="Last time a reverse lookup was run against the IP.", + ) + from_cidr = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field that flags if the IP came from a stakeholder provided cidr.", + ) + + # domains = models.ManyToManyField("SubDomains", related_name='ips', blank=True) + # host_scans = models.ManyToManyField("HostScan", related_name='ips', blank=True) + # hosts = models.ManyToManyField(Host, related_name='ips', blank=True) + # tickets = models.ManyToManyField("Ticket", related_name='ips', blank=True) + # vuln_scans = models.ManyToManyField(VulnScan, related_name='ips', blank=True) + # port_scans = models.ManyToManyField("PortScan", related_name='ips', blank=True) + sub_domains = models.ManyToManyField( + "SubDomains", + related_name="ips", + blank=True, + help_text="Many to many relationship linking to sub domains that were seen running on the IP.", + ) + has_shodan_results = models.BooleanField( + blank=True, + null=True, + help_text="A boolean field that flags if shodan has findings for the givenn IP", + ) + origin_cidr = models.ForeignKey( + Cidr, + on_delete=models.CASCADE, + db_column="origin_cidr", + blank=True, + null=True, + help_text="Foreign key relationship to the cidr from which the ip was enumerated.", + ) + current = models.BooleanField( + blank=True, + null=True, + help_text="A boolean field that flags if the IP is current.", + ) + + class Meta: + """The Meta class for Ip.""" + + app_label = app_label_name + managed = manage_db + db_table = "ip" + indexes = [models.Index(fields=["ip", "organization"])] + unique_together = ["ip", "organization"] + + +class Ticket(models.Model): + """The Ticket model.""" + + id = models.CharField( + max_length=255, + primary_key=True, + help_text="Unique identifier for a ticket object in the database.", + ) # Assuming the UUID is represented as a string + cve_string = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="CVE (Common Vulnerabilities and Exposures) id for the vulnerability.", + ) + cve = models.ForeignKey( + Cve, + related_name="tickets", + null=True, + blank=True, + on_delete=models.CASCADE, + help_text="Foreign key relationship to the related CVE object.", + ) + cvss_base_score = models.DecimalField( + max_digits=5, + decimal_places=2, + null=True, + blank=True, + help_text="CVSS base score](https://nvd.nist.gov/vuln-metrics)", + ) + cvss_version = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="CVSS version used for the CVSS base score", + ) + # kev = models.ForeignKey(Kev, related_name='tickets', null=True, blank=True, on_delete=models.CASCADE) + vuln_name = models.CharField( + max_length=255, null=True, blank=True, help_text="Vulnerability name" + ) + cvss_score_source = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Source of the CVSS base score (e.g. 'nvd' or 'nessus')", + ) + cvss_severity = models.DecimalField( + max_digits=5, + decimal_places=2, + null=True, + blank=True, + help_text="[CVSS severity rating](https://nvd.nist.gov/vuln-metrics)", + ) + vpr_score = models.DecimalField( + max_digits=5, + decimal_places=2, + null=True, + blank=True, + help_text="Tenable [Vulnerability Priority Rating](https://docs.tenable.com/nessus/Content/RiskMetrics.htm)", + ) + false_positive = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if this ticket is marked as a false positive?", + ) + ip_string = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="IP address of the host that was vulnerability scanned", + ) + ip = models.ForeignKey( + Ip, + related_name="tickets", + null=True, + blank=True, + on_delete=models.CASCADE, + help_text="Foreign key relationship to the related IP object.", + ) + updated_timestamp = models.DateTimeField( + null=True, blank=True, help_text="Timestamp of when the ticket was last updated" + ) + location_longitude = models.DecimalField( + max_digits=9, + decimal_places=6, + null=True, + blank=True, + help_text="Longitude of host (according to geolocation database) associated with this ticket", + ) + location_latitude = models.DecimalField( + max_digits=9, + decimal_places=6, + null=True, + blank=True, + help_text="Latitude of host (according to geolocation database) associated with this ticket", + ) + found_in_latest_host_scan = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if this vulnerability was detected in the latest scan of the associated host?", + ) + organization = models.ForeignKey( + Organization, + related_name="tickets", + null=True, + blank=True, + on_delete=models.CASCADE, + help_text="Foreign key relationship to the organization that owns the asset that was scanned.", + ) + vuln_port = models.IntegerField( + null=True, blank=True, help_text="Number of the vulnerable port in this ticket." + ) + port_protocol = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Protocol for the vulnerable port in this ticket ('tcp' or 'udp')", + ) + snapshots_bool = models.BooleanField( + null=True, + blank=True, + help_text="Boolean field that flags if there are snapshots that include this ticket", + ) + vuln_source = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Source of the vulnerability scan (e.g. 'nessus' or 'nmap')", + ) + vuln_source_id = models.IntegerField( + null=True, + blank=True, + help_text="Source-specific identifier for the vulnerability scan (e.g. the scanner plugin identifier that detected the vulnerability)", + ) + closed_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp when this ticket was closed (vulnerability was no longer detected); value of null indicates that this ticket is currently open", + ) + opened_timestamp = models.DateTimeField( + null=True, + blank=True, + help_text="Timestamp when this ticket was opened (vulnerability was first detected)", + ) + # snapshots = models.ManyToManyField(Snapshot, related_name='tickets', blank=True) + # ticket_events = models.ManyToManyField(TicketEvent, related_name='tickets', blank=True) + + class Meta: + """The Meta class for Ticket.""" + + app_label = app_label_name + managed = manage_db + db_table = "ticket" + unique_together = ["id"] + + +class PortScan(models.Model): + """The PortScan model.""" + + id = models.CharField( + max_length=255, + primary_key=True, + help_text="Unique identifier for the port scan object.", + ) # Assuming UUIDs are stored as strings + ip_string = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="IP address of the host that was port scanned", + ) + ip = models.ForeignKey( + Ip, + related_name="port_scans", + null=True, + blank=True, + on_delete=models.CASCADE, + help_text="Foreign key relationship to the related IP.", + ) + latest = models.BooleanField( + default=False, + help_text="Booolean field that flags if this is the latest scan of this port.", + ) + port = models.IntegerField( + null=True, blank=True, help_text="Number of the port that was scanned." + ) + protocol = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Protocol for this port scan ('tcp' or 'udp').", + ) + reason = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Why this port is determined to be open, as reported by the port scanner.", + ) + service = models.JSONField( + default=dict, help_text="Details about this port, as reported by the scanner" + ) # Use JSONField to store JSON objects + service_name = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Source of the scan (e.g. 'nmap')", + ) + service_confidence = models.IntegerField( + null=True, blank=True, help_text="Level of confidence the service is running." + ) + service_method = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="The method that was used to identify the service on the port.", + ) + source = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Source of the scan (e.g. 'nmap')", + ) + state = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="State of the port, as reported by the scanner; see nmap states", + ) + time_scanned = models.DateTimeField( + null=True, blank=True, help_text="Timestamp when the port was scanned" + ) + # snapshots = models.ManyToManyField(Snapshot, related_name='port_scans', blank=True) + organization = models.ForeignKey( + Organization, + related_name="port_scans", + null=True, + blank=True, + on_delete=models.CASCADE, + help_text="Foreign key relationship to the organization that owns the scanned IP.", + ) + + class Meta: + """The Meta class for PortScan.""" + + app_label = app_label_name + managed = manage_db + db_table = "port_scan" + + +# ####### WAS Models ######### + + +class WasTrackerCustomerdata(models.Model): + """Define WasTrackerCustomerdata model.""" + + customer_id = models.UUIDField( + db_column="customer_id", + primary_key=True, + default=uuid.uuid1, + help_text="Unique identifier for a Was customer.", + ) + tag = models.TextField( + help_text="Short name of the customer used to query reports, ideally shoulud match ServiceNow, PE and VS." + ) + customer_name = models.TextField(help_text="Full name of the WAS customer.") + testing_sector = models.TextField(help_text="The sector the customer falls under.") + ci_type = models.TextField(help_text="Critical infrastructure classification.") + jira_ticket = models.TextField(help_text="???") + ticket = models.TextField(help_text="???") + next_scheduled = models.TextField( + help_text="The next date and time the customer's webapps will be scanned." + ) + last_scanned = models.TextField( + help_text="The last date and time the customer's webapps were scanned." + ) + frequency = models.TextField(help_text="The frequency the WAS reports are run.") + comments_notes = models.TextField( + help_text="Additional comments and notes about how and when to run the report." + ) + was_report_poc = models.TextField(help_text="Customer's point of contact.") + was_report_email = models.TextField( + help_text="Email address(es) that WAS reports are delivered to." + ) + onboarding_date = models.TextField( + help_text="Date that the customer was added to the WAS service." + ) + no_of_web_apps = models.IntegerField( + help_text="Number of webapps the customer has submitted to be scanned." + ) + no_web_apps_last_updated = models.TextField( + blank=True, + null=True, + help_text="The last datetime that the number of apps was updated.", + ) + elections = models.BooleanField( + blank=False, + null=False, + help_text="Boolean field that flags if the customer is an election entity.", + ) + fceb = models.BooleanField( + blank=False, + null=False, + help_text="Boolean field that flags if the customer is an FCEB entity.", + ) + special_report = models.BooleanField( + blank=False, + null=False, + help_text="Boolean field that flags if the customer receives a special report.", + ) + report_password = models.TextField( + help_text="The password used to encrypt the WAS report." + ) + child_tags = models.TextField(help_text="List of tags of any child customers.") + + class Meta: + """Set WasTrackerCustomerdata model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "was_tracker_customer_data" + + +""" +-- WARNING: It may differ from actual native database DDL +CREATE TABLE information_schema.was_findings ( + finding_uid uuid NOT NULL, + finding_type varchar(10485760) NULL, + webapp_id int4 NULL, + was_org_id text NULL, + owasp_category varchar(10485760) NULL, + severity varchar(10485760) NULL, + times_detected int4 NULL, + base_score float8 NULL, + temporal_score float8 NULL, + fstatus varchar(10485760) NULL, + last_detected date NULL, + first_detected date NULL, + is_remediated bool NULL, + potential bool NULL, + webapp_url text NULL, + webapp_name text NULL, + "name" text NULL, + cvss_v3_attack_vector text NULL, + cwe_list _int4 NULL, + wasc_list jsonb NULL +); +""" + + +class WasFindings(models.Model): + """Define WasFindings model.""" + + finding_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="Unique identifier for a WAS finding object.", + ) + finding_type = models.TextField( + blank=True, + null=True, + help_text="Type of WAS finding. (INFORMATION_GATHERED, SENSITIVE_CONTENT, VULNERABILITY)", + ) + webapp_id = models.IntegerField( + blank=True, + null=True, + help_text="Identifier for the webapp on which the finding was found.", + ) + was_org_id = models.TextField( + blank=True, + null=True, + help_text="Acronym of the customer who owns the scanned webapp.", + ) + owasp_category = models.TextField( + blank=True, + null=True, + help_text="OWASP (Open Web Application Security Project) categorization of the finding.", + ) + severity = models.TextField( + blank=True, null=True, help_text="Severity of the finding, rated 1-5." + ) + times_detected = models.IntegerField( + blank=True, null=True, help_text="How many times the finding has been seen." + ) + base_score = models.FloatField( + blank=True, null=True, help_text="Base CVSS score for the finding." + ) + temporal_score = models.FloatField( + blank=True, null=True, help_text="Temporal CVSS score for the finding." + ) + fstatus = models.TextField( + blank=True, + null=True, + help_text="Status of finding. (NEW, ACTIVE, REOPENED, FIXED)", + ) + last_detected = models.DateField( + blank=True, null=True, help_text="The last time the finding was seen." + ) + first_detected = models.DateField( + blank=True, null=True, help_text="The first time the finding was seen." + ) + is_remediated = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the fiding has been remediated.", + ) + potential = models.BooleanField(blank=True, null=True, help_text="???") + webapp_url = models.TextField( + blank=True, + null=True, + help_text="URL of the webapp where the finding was identified.", + ) + webapp_name = models.TextField( + blank=True, + null=True, + help_text="Name of the webapp where the finding was identified.", + ) + name = models.TextField(blank=True, null=True, help_text="Name of the finding.") + cvss_v3_attack_vector = models.TextField( + blank=True, + null=True, + help_text="Vector of the attack. (Adjacent Network, Local Access, Network, None)", + ) + cwe_list = ArrayField( + models.IntegerField(blank=True, null=True), + blank=True, + null=True, + help_text="List of CWEs identified in the finding.", + ) + wasc_list = models.JSONField( + blank=True, + null=True, + help_text="List of dictionaries containing links to relevant WASC (Web Application Security Consortium) references.", + ) + last_tested = models.DateField( + blank=True, null=True, help_text="Last time the finding was tested." + ) + fixed_date = models.DateField( + blank=True, null=True, help_text="Date the finding was remediated." + ) + is_ignored = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the customer has decided to ignore the finding.", + ) + url = models.TextField( + blank=True, null=True, help_text="URL where the finding was identified." + ) + qid = models.IntegerField( + blank=True, null=True, help_text="Qualys id for the finding." + ) + response = models.TextField( + blank=True, null=True, help_text="The returned response from the webapp." + ) + + class Meta: + """Set WasFindings model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "was_findings" + + +class WasHistory(models.Model): + """Define WasHistory model.""" + + was_org_id = models.TextField( + blank=True, null=True, help_text="Unique identifier for the WAS history object." + ) + date_scanned = models.DateField( + help_text="The date a customers webapps were scanned." + ) + vuln_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many vulnerabilities were identified.", + ) + vuln_webapp_cnt = models.IntegerField( + blank=True, null=True, help_text="The count of how many webapps are vulnerable." + ) + web_app_cnt = models.IntegerField( + blank=True, + null=True, + help_text="Count of how many webapps were scanned for the customer.", + ) + high_rem_time = models.IntegerField( + blank=True, + null=True, + help_text="Average time it took to remediate vulnerabilities with a high severity.", + ) + crit_rem_time = models.IntegerField( + blank=True, + null=True, + help_text="Average time it took to remediate vulnerabilities with a critical severity.", + ) + crit_vuln_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many vulnerabilites have a critical severity.", + ) + high_vuln_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many vulnerabilites have a high severity.", + ) + report_period = models.DateField( + blank=True, + null=True, + help_text="The report period these findings were identified within.", + ) + high_rem_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many high severity vulnerabiliteis were remediated.", + ) + crit_rem_cnt = models.IntegerField( + blank=True, + null=True, + help_text="A count of how many critical severity vulnerabiliteis were remediated.", + ) + total_potential = models.IntegerField( + blank=True, + null=True, + help_text="A count of all potential vulnerabilities there are across a customer's webapps.", + ) + + class Meta: + """Set WasHistory model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "was_history" + unique_together = (("was_org_id", "date_scanned"),) + + +class WasMap(models.Model): + """Define WasMap model.""" + + was_org_id = models.TextField( + blank=True, primary_key=True, help_text="WAS customer acronym." + ) + pe_org_id = models.UUIDField( + blank=True, null=True, help_text="Corresponding PE organization acronym" + ) + report_on = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the organization receives a report???", + ) # Not sure if this is a PE or WAS report. + last_scanned = models.DateField( + blank=True, + null=True, + help_text="Last time the organization was scanned by WAS.", + ) + + class Meta: + """Set WasMap model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "was_map" + + +class WasReport(models.Model): + """The WasReport model.""" + + org_name = models.TextField( + blank=True, null=True, help_text="Name of the WAS customer." + ) + date_pulled = models.DateTimeField( + blank=True, null=True, help_text="Date the was report wasw pulled from Qualys." + ) + last_scan_date = models.DateTimeField( + blank=True, + null=True, + help_text="Last time WAS ran a Qualys scan on the organization's webapps.", + ) + security_risk = models.TextField( + blank=True, null=True, help_text="Security risk the customer's webapps face." + ) + total_info = models.IntegerField( + blank=True, + null=True, + help_text="Number of findings found across the customer's webapps.???", + ) + num_apps = models.IntegerField( + blank=True, null=True, help_text="Number of webapps scanned in this report." + ) + risk_color = models.TextField( + blank=True, + null=True, + help_text="Color code for the risk level presented in the reports.", + ) + sensitive_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of sensitive findings included inn the report.???", + ) + sensitive_color = models.TextField( + blank=True, + null=True, + help_text="Color code for the sensitivity level presented in the report.", + ) + max_days_open_urgent = models.IntegerField( + blank=True, + null=True, + help_text="The maximum days an urgent finding has remained open.", + ) + max_days_open_critical = models.IntegerField( + blank=True, + null=True, + help_text="The maximum days a critical finding has remained open.", + ) + urgent_color = models.TextField( + blank=True, + null=True, + help_text="Color code used to display urgent details in the report.", + ) + critical_color = models.TextField( + blank=True, + null=True, + help_text="Color code used to display critical details in the report.", + ) + org_was_acronym = models.TextField( + blank=True, + null=True, + help_text="Acronym or short name of the organization receiving the report.", + ) + name_len = models.TextField( + blank=True, + null=True, + help_text="Number of characters in the organization's name.", + ) + vuln_csv_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary containing counts of each type of vulnerability for each webapp", + ) + ssn_cc_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary containing counts of credit card and social security data found on the customer's webapps.", + ) + app_overview_csv_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary containing an overview of the apps urls and findings.", + ) + details_csv = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of additional details regarding the findings in the report.???", + ) + info_csv = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of Finding information for each of the findings in the report.", + ) + links_crawled = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of links crawled including duration of time and depth of the crawl.", + ) + links_rejected = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of rejecting links and which webapp they were from.???", + ) + emails_found = models.JSONField( + blank=True, + null=True, + default=list, + help_text="List of emails found in each webapp.", + ) + owasp_count_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary that counts each of the OWASP categories for each webapp.???", + ) + group_count_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary counting the sums of each OWASP category for all webapps.", + ) + fixed = models.IntegerField( + blank=True, + null=True, + help_text="Count of fixed vulns in all webapps owned by the organization.", + ) + total = models.IntegerField( + blank=True, + null=True, + help_text="Total vulnerability count across all webapps owned by the organization.", + ) + vulns_monthly_dict = models.JSONField( + blank=True, + null=True, + default=dict, + help_text="Dictionary summing each of the findings by month they were found.", + ) + path_disc = models.IntegerField(blank=True, null=True, help_text="???") + info_disc = models.IntegerField(blank=True, null=True, help_text="???") + cross_site = models.IntegerField( + blank=True, + null=True, + help_text="Count of cross site scripting vulnerabilities.", + ) + burp = models.IntegerField( + blank=True, null=True, help_text="Vulnerabilities detected by BURP.???" + ) + sql_inj = models.IntegerField( + blank=True, null=True, help_text="Count of SQL injection vulnerabilities." + ) + bugcrowd = models.IntegerField( + blank=True, null=True, help_text="Vulnerabilities detected by Bugcrowd." + ) + reopened = models.IntegerField( + blank=True, null=True, help_text="Count of reopened vulnerabilities." + ) + reopened_color = models.TextField( + blank=True, null=True, help_text="Color used to display the reopened count." + ) + new_vulns = models.IntegerField( + blank=True, null=True, help_text="Count of new vulnerablities" + ) + new_vulns_color = models.TextField( + blank=True, + null=True, + help_text="Color code used to display count of new vulnerabilities.", + ) + tot_vulns = models.IntegerField( + blank=True, null=True, help_text="Total count of vulnerabilities." + ) + tot_vulns_color = models.TextField( + blank=True, + null=True, + help_text="Color code used to display count of new vulnerabilities.", + ) + lev1 = models.IntegerField( + blank=True, null=True, help_text="Count of level 1 vulnerabilities." + ) + lev2 = models.IntegerField( + blank=True, null=True, help_text="Count of level 2 vulnerabilities." + ) + lev3 = models.IntegerField( + blank=True, null=True, help_text="Count of level 3 vulnerabilities." + ) + lev4 = models.IntegerField( + blank=True, null=True, help_text="Count of level 4 vulnerabilities." + ) + lev5 = models.IntegerField( + blank=True, null=True, help_text="Count of level 5 vulnerabilities." + ) + severities = ArrayField( + models.IntegerField(), + blank=True, + null=True, + default=list, + help_text="List of the severities assigned to each of the vulnerabilities.", + ) + ages = ArrayField( + models.IntegerField(), + blank=True, + null=True, + default=list, + help_text="List of ages of all the vulnerabilities.", + ) + pdf_obj = models.BinaryField( + blank=True, + null=True, + help_text="PDF binary or the full pdf generated by Qualys.", + ) + + class Meta: + """The Meta class for WasReport.""" + + db_table = "was_report" + unique_together = ("last_scan_date", "org_was_acronym") + app_label = app_label_name + managed = manage_db + + +# ######## PE Models ######### +class PeUsers(models.Model): + """Define Users model.""" + + id = models.UUIDField( + primary_key=True, help_text="Unique identifier for a PE user object." + ) + email = models.CharField( + unique=True, + max_length=64, + blank=True, + null=True, + help_text="Email address of the user.", + ) + username = models.CharField( + unique=True, + max_length=64, + blank=True, + null=True, + help_text="Username of the user.", + ) + admin = models.IntegerField( + blank=True, + null=True, + help_text="Django generated field that determines the admin permissions for the user.", + ) + role = models.IntegerField( + blank=True, + null=True, + help_text="Django generated field that determines the role permissions for the user.", + ) + password_hash = models.CharField( + max_length=128, + blank=True, + null=True, + help_text="Cryptographic hash of the user's password.", + ) + api_key = models.CharField( + unique=True, + max_length=128, + blank=True, + null=True, + help_text="The user's API key.", + ) + + class Meta: + """Set User model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "pe_users" + + +# ?????? not sure if we use this anywhere +class AlembicVersion(models.Model): + """Define AlembicVersion model.""" + + version_num = models.CharField( + primary_key=True, + max_length=32, + help_text="A unique identifier assigned to each database migration script in Alembic, this may not be used currently.", + ) + + class Meta: + """Set AlembicVersion model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "alembic_version" + + +class SixgillAlerts(models.Model): + """Define Alerts model.""" + + alerts_uid = models.UUIDField( + primary_key=True, + help_text="Unique identifier for the cyber sixgill alert object.", + ) + alert_name = models.TextField( + blank=True, null=True, help_text="Name of the alert provided by Cybersixgill." + ) + content = models.TextField( + blank=True, + null=True, + help_text="Content of the post or website that triggered the alert.", + ) + date = models.DateField( + blank=True, null=True, help_text="Date the alert was created." + ) + sixgill_id = models.TextField( + unique=True, + blank=True, + null=True, + help_text="Cybersixgill ID associated with alert.", + ) + read = models.TextField( + blank=True, + null=True, + help_text="Boolean field that flags if the alert was read in the Cybersixgill portal.", + ) + severity = models.TextField( + blank=True, null=True, help_text="Severity ranking of alert from 1 - 10." + ) + site = models.TextField( + blank=True, null=True, help_text="Site associated with the alert." + ) + threat_level = models.TextField( + blank=True, + null=True, + help_text="Threat level of alert either 'imminent' or 'emerging'.", + ) + threats = models.TextField( + blank=True, null=True, help_text="Type of threat for alert" + ) + title = models.TextField(blank=True, null=True, help_text="Title of alert post") + user_id = models.TextField( + blank=True, null=True, help_text="Id of user that made the API call" + ) + category = models.TextField(blank=True, null=True, help_text="Category of alert") + lang = models.TextField( + blank=True, null=True, help_text="Language of alert content" + ) + organization = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="Foreign Key to the related organization", + ) + data_source = models.ForeignKey( + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="Foreign Key to the data_source.", + ) + content_snip = models.TextField( + blank=True, + null=True, + help_text="100 character snippet of the post content. 50 characters before/after the specific mention", + ) + asset_mentioned = models.TextField( + blank=True, null=True, help_text="Asset mentioned in alert" + ) + asset_type = models.TextField( + blank=True, null=True, help_text="Type of asset mentioned in alert" + ) + + class Meta: + """Set Alerts model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "sixgill_alerts" + + +class Alias(models.Model): + """Define Alias model.""" + + alias_uid = models.UUIDField( + primary_key=True, help_text="Unique identifier for an alias." + ) + organization = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organizations", + ) + alias = models.TextField(unique=True, help_text="Alias for an organization") + + class Meta: + """Set Alias model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "alias" + + +# ?????? +class AssetHeaders(models.Model): + """Define AssetHeaders model.""" + + field_id = models.UUIDField( + db_column="_id", + primary_key=True, + help_text="Unique identifier for the asset header object.", + ) # Field renamed because it started with '_'. + organization = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="Foreign key relationship to the organization that owns the asset.", + ) + sub_url = models.TextField(help_text="URL to the subdomain that was scanned.") + tech_detected = models.TextField( + help_text="List of technologies identified running on the subdomain." + ) # This field type is a guess. + interesting_header = models.TextField( + help_text="List of headers that potentially have relevant findings." + ) # This field type is a guess. + ssl2 = models.TextField( + blank=True, + null=True, + help_text="Evidence that the subdomain is running the outdateed SSL2 protocol", + ) # This field type is a guess. + tls1 = models.TextField( + blank=True, + null=True, + help_text="Evidence that the subdomain is running the outdateed TLS1 protocol", + ) # This field type is a guess. + certificate = models.TextField( + blank=True, null=True, help_text="Certificate details of the subdomain." + ) # This field type is a guess. + scanned = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the suubdomain has been scanned.", + ) + ssl_scanned = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if an SSL scan has been run against the subdomain.", + ) + + class Meta: + """Set AssetHeaders model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "asset_headers" + unique_together = (("organization", "sub_url"),) + + +# # ?????? no data currently +# class AuthGroup(models.Model): +# """Define AuthGroup model.""" + +# name = models.CharField(unique=True, max_length=150) + +# class Meta: +# """Set AuthGroup model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_group" + +# # ?????? no data currently +# class AuthGroupPermissions(models.Model): +# """Define AuthGroupPermissions model.""" + +# id = models.BigAutoField(primary_key=True) +# group = models.ForeignKey(AuthGroup, on_delete=models.CASCADE) +# permission = models.ForeignKey("AuthPermission", on_delete=models.CASCADE) + +# class Meta: +# """Set AuthGroupPermissions model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_group_permissions" +# unique_together = (("group", "permission"),) + +# # ?????? +# class AuthPermission(models.Model): +# """Define AuthPermission model.""" +# id = models.BigAutoField(primary_key=True) +# name = models.CharField(max_length=255) +# content_type = models.ForeignKey("DjangoContentType", on_delete=models.CASCADE) +# codename = models.CharField(max_length=100) + +# class Meta: +# """Set AuthPermission model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_permission" +# unique_together = (("content_type", "codename"),) + +# # ?????? +# class AuthUser(models.Model): +# """Define AuthUser model.""" +# id = models.BigAutoField(primary_key=True) +# password = models.CharField(max_length=128) +# last_login = models.DateTimeField(blank=True, null=True) +# is_superuser = models.BooleanField() +# username = models.CharField(unique=True, max_length=150) +# first_name = models.CharField(max_length=150) +# last_name = models.CharField(max_length=150) +# email = models.CharField(max_length=254) +# is_staff = models.BooleanField() +# is_active = models.BooleanField() +# date_joined = models.DateTimeField() + +# class Meta: +# """Set AuthUser model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_user" + +# # ?????? currently empty +# class AuthUserGroups(models.Model): +# """Define AuthUserGroups model.""" + +# id = models.BigAutoField(primary_key=True) +# user = models.ForeignKey(AuthUser, on_delete=models.CASCADE) +# group = models.ForeignKey(AuthGroup, on_delete=models.CASCADE) + +# class Meta: +# """Set AuthUserGroups model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_user_groups" +# unique_together = (("user", "group"),) + +# # ?????? currently empty +# class AuthUserUserPermissions(models.Model): +# """Define AuthUserUserPermissions model.""" + +# id = models.BigAutoField(primary_key=True) +# user = models.ForeignKey(AuthUser, on_delete=models.CASCADE) +# permission = models.ForeignKey(AuthPermission, on_delete=models.CASCADE) + +# class Meta: +# """Set AuthUserUserPermissions model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "auth_user_user_permissions" +# unique_together = (("user", "permission"),) + + +class CredentialBreaches(models.Model): + """Define CredentialBreaches model.""" + + credential_breaches_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for credential_breaches.", + ) + breach_name = models.TextField(unique=True, help_text="Name of breach.") + description = models.TextField( + blank=True, null=True, help_text="Description of breach." + ) + exposed_cred_count = models.BigIntegerField( + blank=True, null=True, help_text="Number of credentials exposed in breach." + ) + breach_date = models.DateField( + blank=True, null=True, help_text="Date when breach occured." + ) + added_date = models.DateTimeField( + blank=True, null=True, help_text="Date breach was added by the source." + ) + modified_date = models.DateTimeField( + blank=True, + null=True, + help_text="Date breach information was last modified/updated.", + ) + data_classes = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="List of types of data identified in the breach.", + ) # This field type is a guess. + password_included = models.BooleanField( + blank=True, + null=True, + help_text="T/F Were passwords included with the credentials?", + ) + is_verified = models.BooleanField( + blank=True, null=True, help_text="T/F Is breach verified?" + ) + is_fabricated = models.BooleanField( + blank=True, null=True, help_text="T/F Is the breach fabricated?" + ) + is_sensitive = models.BooleanField( + blank=True, + null=True, + help_text="T/F Does the breach contain sensitive content?", + ) + is_retired = models.BooleanField( + blank=True, + null=True, + help_text="T/F Has the breach been retired? (I believe the means it is no longer posted", + ) + is_spam_list = models.BooleanField( + blank=True, null=True, help_text="T/F Is the breach a spam list?" + ) + data_source = models.ForeignKey( + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + blank=True, + null=True, + help_text="FK: Foreign Key to data_source", + ) + + class Meta: + """Set CredentialBreaches model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "credential_breaches" + + +class CredentialExposures(models.Model): + """Define CredentialExposures model.""" + + credential_exposures_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for credential_exposures", + ) + email = models.TextField(help_text="Email found in the breach") + organization = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organization", + ) + root_domain = models.TextField( + blank=True, + null=True, + help_text="The root domain for the email found in the breach", + ) + sub_domain = models.TextField( + blank=True, + null=True, + help_text="The sub domain for thee email found in the breach", + ) + breach_name = models.TextField( + blank=True, null=True, help_text="Name of breach where credentials were exposed" + ) + modified_date = models.DateTimeField( + blank=True, + null=True, + help_text="Date credential exposure information was last modified/updated", + ) + credential_breaches = models.ForeignKey( + CredentialBreaches, + on_delete=models.CASCADE, + db_column="credential_breaches_uid", + help_text="FK: Foreign Key to credential_breaches", + ) + data_source = models.ForeignKey( + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + blank=True, + null=True, + help_text="FK: Foreign Key to data_source", + ) + name = models.TextField( + blank=True, null=True, help_text="Name of person whose credentials were exposed" + ) + login_id = models.TextField( + blank=True, + null=True, + help_text="Login ID of person whose credentials were exposed", + ) + phone = models.TextField( + blank=True, + null=True, + help_text="Phone number of person whose credentials were exposed", + ) + password = models.TextField( + blank=True, + null=True, + help_text="Password of person whose credentials were exposed", + ) + hash_type = models.TextField( + blank=True, null=True, help_text="The method used to hash the password" + ) + intelx_system_id = models.TextField( + blank=True, null=True, help_text="Id of the Exposure in the intelx system." + ) + + class Meta: + """Set CredentialExposures model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "credential_exposures" + unique_together = (("breach_name", "email"),) + + +class CyhyContacts(models.Model): + """Define CyhyContacts model.""" + + field_id = models.UUIDField( + db_column="_id", + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for cyhy contacts", + ) # Field renamed because it started with '_'. + org_id = models.TextField(help_text="Organization abbreviated name") + organization = models.ForeignKey( + "Organization", + models.DO_NOTHING, + db_column="organization_uid", + help_text="FK: Foreign key to the organization", + ) + org_name = models.TextField(help_text="Organization full name") + phone = models.TextField( + blank=True, null=True, help_text="Phone number for organization contact" + ) + contact_type = models.TextField(help_text="Type of contact") + email = models.TextField( + blank=True, null=True, help_text="Email for organization contact" + ) + name = models.TextField( + blank=True, null=True, help_text="Name of organization contact" + ) + date_pulled = models.DateField( + blank=True, + null=True, + help_text="The date we pulled the contact from the cyhy database more recently", + ) + + class Meta: + """Set CyhyContacts model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "cyhy_contacts" + unique_together = (("org_id", "contact_type", "email", "name"),) + + +class CyhyDbAssets(models.Model): + """Define CyhyDbAssets model.""" + + field_id = models.UUIDField( + db_column="_id", + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for cyhy db assets", + ) # Field renamed because it started with '_'. + org_id = models.TextField( + blank=True, null=True, help_text="Organization abbreviated name" + ) + organization = models.ForeignKey( + "Organization", + models.DO_NOTHING, + db_column="organization_uid", + help_text="FK: Foreign key to the organization", + ) + org_name = models.TextField( + blank=True, null=True, help_text="Organization full name" + ) + contact = models.TextField( + blank=True, null=True, help_text="Organization contact information" + ) + network = models.GenericIPAddressField( + blank=True, + null=True, + help_text="Cidr range or IP address owned by the organization", + ) + type = models.TextField(blank=True, null=True, help_text="Network type") + first_seen = models.DateField( + blank=True, + null=True, + help_text="First date and time the asset was associated with the cyhy customer.", + ) + last_seen = models.DateField( + blank=True, + null=True, + help_text="Last date and time the asset was associated with the cyhy customer.", + ) + currently_in_cyhy = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field flagging if the cidr was seen in the last pull from cyhy", + ) + + class Meta: + """Set CyhyDbAssets model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "cyhy_db_assets" + unique_together = (("org_id", "network"),) + + +# TODO determine if we want user logic on both databases +# class PEDataapiApiuser(models.Model): +# """Define DataapiApiuser model.""" + +# id = models.BigAutoField(primary_key=True) +# apikey = models.CharField( +# db_column="apiKey", max_length=200, blank=True, null=True +# ) # Field name made lowercase. +# user = models.OneToOneField(AuthUser, on_delete=models.CASCADE) +# refresh_token = models.CharField(max_length=200, blank=True, null=True) + +# class Meta: +# """Set DataapiApiuser model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "pe_dataAPI_apiuser" + + +# ?????? +class DataSource(models.Model): + """Define DataSource model.""" + + data_source_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for data_sources", + ) + name = models.TextField(help_text="Name of data source") + description = models.TextField(help_text="Description of data source") + last_run = models.DateField(help_text="Date that data source was last ran") + + class Meta: + """Set DataSource model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "data_source" + + +# # ?????? +# class DjangoAdminLog(models.Model): +# """Define DjangoAdminLog model.""" +# id = models.BigAutoField(primary_key=True) +# action_time = models.DateTimeField() +# object_id = models.TextField(blank=True, null=True) +# object_repr = models.CharField(max_length=200) +# action_flag = models.SmallIntegerField() +# change_message = models.TextField() +# content_type = models.ForeignKey( +# "DjangoContentType", on_delete=models.CASCADE, blank=True, null=True +# ) +# user = models.ForeignKey(AuthUser, on_delete=models.CASCADE) + +# class Meta: +# """Set DjangoAdminLog model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "django_admin_log" + +# # ?????? +# class DjangoContentType(models.Model): +# """Define DjangoContentType model.""" +# id = models.BigAutoField(primary_key=True) +# app_label = models.CharField(max_length=100) +# model = models.CharField(max_length=100) + +# class Meta: +# """Set DjangoContentType model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "django_content_type" +# unique_together = (("app_label", "model"),) + +# # ?????? +# class DjangoMigrations(models.Model): +# """Define DjangoMigrations model.""" + +# id = models.BigAutoField(primary_key=True) +# app = models.CharField(max_length=255) +# name = models.CharField(max_length=255) +# applied = models.DateTimeField() + +# class Meta: +# """Set DjangoMigrations model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "django_migrations" + +# # ?????? +# class DjangoSession(models.Model): +# """Define DjangoSession model.""" + +# session_key = models.CharField(primary_key=True, max_length=40) +# session_data = models.TextField() +# expire_date = models.DateTimeField() + +# class Meta: +# """Set DjangoSession model metadata.""" + +# app_label = 'dmz_mini_dl' +# managed = manage_db +# db_table = "django_session" + + +class DnsRecords(models.Model): + """Define DnsRecords model.""" + + dns_record_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="Unique identifier for a DNS record.", + ) + domain_name = models.TextField(blank=True, null=True, help_text="") + domain_type = models.TextField(blank=True, null=True, help_text="") + created_date = models.DateTimeField(blank=True, null=True, help_text="") + updated_date = models.DateTimeField(blank=True, null=True, help_text="") + expiration_date = models.DateTimeField(blank=True, null=True, help_text="") + name_servers = models.TextField( + blank=True, null=True, help_text="" + ) # This field type is a guess. + whois_server = models.TextField(blank=True, null=True, help_text="") + registrar_name = models.TextField(blank=True, null=True, help_text="") + status = models.TextField(blank=True, null=True, help_text="") + clean_text = models.TextField(blank=True, null=True, help_text="") + raw_text = models.TextField(blank=True, null=True, help_text="") + registrant_name = models.TextField(blank=True, null=True, help_text="") + registrant_organization = models.TextField(blank=True, null=True, help_text="") + registrant_street = models.TextField(blank=True, null=True, help_text="") + registrant_city = models.TextField(blank=True, null=True, help_text="") + registrant_state = models.TextField(blank=True, null=True, help_text="") + registrant_post_code = models.TextField(blank=True, null=True, help_text="") + registrant_country = models.TextField(blank=True, null=True, help_text="") + registrant_email = models.TextField(blank=True, null=True, help_text="") + registrant_phone = models.TextField(blank=True, null=True, help_text="") + registrant_phone_ext = models.TextField(blank=True, null=True, help_text="") + registrant_fax = models.TextField(blank=True, null=True, help_text="") + registrant_fax_ext = models.TextField(blank=True, null=True, help_text="") + registrant_raw_text = models.TextField(blank=True, null=True, help_text="") + administrative_name = models.TextField(blank=True, null=True, help_text="") + administrative_organization = models.TextField(blank=True, null=True, help_text="") + administrative_street = models.TextField(blank=True, null=True, help_text="") + administrative_city = models.TextField(blank=True, null=True, help_text="") + administrative_state = models.TextField(blank=True, null=True, help_text="") + administrative_post_code = models.TextField(blank=True, null=True, help_text="") + administrative_country = models.TextField(blank=True, null=True, help_text="") + administrative_email = models.TextField(blank=True, null=True, help_text="") + administrative_phone = models.TextField(blank=True, null=True, help_text="") + administrative_phone_ext = models.TextField(blank=True, null=True, help_text="") + administrative_fax = models.TextField(blank=True, null=True, help_text="") + administrative_fax_ext = models.TextField(blank=True, null=True, help_text="") + administrative_raw_text = models.TextField(blank=True, null=True, help_text="") + technical_name = models.TextField(blank=True, null=True, help_text="") + technical_organization = models.TextField(blank=True, null=True, help_text="") + technical_street = models.TextField(blank=True, null=True, help_text="") + technical_city = models.TextField(blank=True, null=True, help_text="") + technical_state = models.TextField(blank=True, null=True, help_text="") + technical_post_code = models.TextField(blank=True, null=True, help_text="") + technical_country = models.TextField(blank=True, null=True, help_text="") + technical_email = models.TextField(blank=True, null=True, help_text="") + technical_phone = models.TextField(blank=True, null=True, help_text="") + technical_phone_ext = models.TextField(blank=True, null=True, help_text="") + technical_fax = models.TextField(blank=True, null=True, help_text="") + technical_fax_ext = models.TextField(blank=True, null=True, help_text="") + technical_raw_text = models.TextField(blank=True, null=True, help_text="") + billing_name = models.TextField(blank=True, null=True, help_text="") + billing_organization = models.TextField(blank=True, null=True, help_text="") + billing_street = models.TextField(blank=True, null=True, help_text="") + billing_city = models.TextField(blank=True, null=True, help_text="") + billing_state = models.TextField(blank=True, null=True, help_text="") + billing_post_code = models.TextField(blank=True, null=True, help_text="") + billing_country = models.TextField(blank=True, null=True, help_text="") + billing_email = models.TextField(blank=True, null=True, help_text="") + billing_phone = models.TextField(blank=True, null=True, help_text="") + billing_phone_ext = models.TextField(blank=True, null=True, help_text="") + billing_fax = models.TextField(blank=True, null=True, help_text="") + billing_fax_ext = models.TextField(blank=True, null=True, help_text="") + billing_raw_text = models.TextField(blank=True, null=True, help_text="") + zone_name = models.TextField(blank=True, null=True, help_text="") + zone_organization = models.TextField(blank=True, null=True, help_text="") + zone_street = models.TextField(blank=True, null=True, help_text="") + zone_city = models.TextField(blank=True, null=True, help_text="") + zone_state = models.TextField(blank=True, null=True, help_text="") + zone_post_code = models.TextField(blank=True, null=True, help_text="") + zone_country = models.TextField(blank=True, null=True, help_text="") + zone_email = models.TextField(blank=True, null=True, help_text="") + zone_phone = models.TextField(blank=True, null=True, help_text="") + zone_phone_ext = models.TextField(blank=True, null=True, help_text="") + zone_fax = models.TextField(blank=True, null=True, help_text="") + zone_fax_ext = models.TextField(blank=True, null=True, help_text="") + zone_raw_text = models.TextField(blank=True, null=True, help_text="") + + class Meta: + """Set DnsRecords model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "dns_records" + + +# Possibly shodan +class DomainAlerts(models.Model): + """Define DomainAlerts model.""" + + domain_alert_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for domain_alerts", + ) + sub_domain = models.ForeignKey( + "SubDomains", + on_delete=models.CASCADE, + db_column="sub_domain_uid", + help_text="FK: Foreign Key to sub_domains", + ) + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + organization_uid = models.UUIDField(help_text="FK: Foreign Key to organizations") + alert_type = models.TextField( + blank=True, null=True, help_text="Type of domain alert" + ) + message = models.TextField( + blank=True, null=True, help_text="Message description associated with alert" + ) + previous_value = models.TextField( + blank=True, null=True, help_text="Previous value associated with alert" + ) + new_value = models.TextField( + blank=True, null=True, help_text="New updated value associated with alert" + ) + date = models.DateField(blank=True, null=True, help_text="Date of alert") + + class Meta: + """Set DomainAlerts model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "domain_alerts" + unique_together = (("alert_type", "sub_domain", "date", "new_value"),) + + +class DomainPermutations(models.Model): + """Define DomainPermutations model.""" + + suspected_domain_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="Unique identifier for a DNSTwist domain permutation.", + ) + organization = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="Foreign Key for a linked Organization", + ) + domain_permutation = models.TextField( + blank=True, + null=True, + help_text="Domain that has been flagged as a possible spoof of a domain owned by the stakeholder.", + ) + ipv4 = models.TextField( + blank=True, null=True, help_text="IPv4 associated with the identified domain" + ) + ipv6 = models.TextField( + blank=True, null=True, help_text="IPv6 associated with the identified domain" + ) + mail_server = models.TextField( + blank=True, null=True, help_text="Mail server seen on the domain." + ) + name_server = models.TextField( + blank=True, null=True, help_text="Name server seen on the domain." + ) + fuzzer = models.TextField(blank=True, null=True, help_text="Fuzzing technique used") + date_observed = models.DateField( + blank=True, null=True, help_text="Date domain permutation was observed" + ) + ssdeep_score = models.TextField( + blank=True, null=True, help_text="HTML similarity with fuzzy hashes" + ) + malicious = models.BooleanField( + blank=True, null=True, help_text="T/F Is subdomain malicious?" + ) + blocklist_attack_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of attacks reported in the Blocklist.de database", + ) + blocklist_report_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of reports reported in the Blocklist.de database", + ) + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + sub_domain = models.ForeignKey( + "SubDomains", + on_delete=models.CASCADE, + db_column="sub_domain_uid", + blank=True, + null=True, + help_text="FK: Foreign Key to sub_domains", + ) + dshield_record_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of records reported in the DSheild database", + ) + dshield_attack_count = models.IntegerField( + blank=True, + null=True, + help_text="Number of attacks reported in the DSHeild databse", + ) + date_active = models.DateField( + blank=True, null=True, help_text="Last known date permutation was active" + ) + + class Meta: + """Set DomainPermutations model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "domain_permutations" + unique_together = (("domain_permutation", "organization"),) + + +class DotgovDomains(models.Model): + """Define DotgovDomains model.""" + + dotgov_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for dotgov_domains", + ) + domain_name = models.TextField(unique=True, help_text="Name of the dotgov domain") + domain_type = models.TextField( + blank=True, null=True, help_text="Branch of govt. for dotgov domain" + ) + agency = models.TextField( + blank=True, null=True, help_text="Name of agency domain is associated with" + ) + organization = models.TextField( + blank=True, + null=True, + help_text="Name of organization domain is associated with", + ) + city = models.TextField(blank=True, null=True, help_text="City of organization") + state = models.TextField(blank=True, null=True, help_text="State of organization") + security_contact_email = models.TextField( + blank=True, null=True, help_text="Email of organization's security contact" + ) + + class Meta: + """Set DotgovDomains model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "dotgov_domains" + + +class Executives(models.Model): + """Define Executives model.""" + + executives_uid = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for executives" + ) + organization = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, + db_column="organization", + help_text="FK: Foreign Key to organizations", + ) + executives = models.TextField(help_text="Executive's name") + + class Meta: + """Set Executives model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "executives" + + +class Mentions(models.Model): + """Define Mentions model.""" + + mentions_uid = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier for cyber sixgill mentions" + ) + category = models.TextField(blank=True, null=True, help_text="Category of mention") + collection_date = models.TextField( + blank=True, null=True, help_text="Date that mention was recorded" + ) + content = models.TextField( + blank=True, null=True, help_text="Content of mention incident" + ) + creator = models.TextField( + blank=True, null=True, help_text="User who created the mention" + ) + date = models.DateField( + blank=True, null=True, help_text="Date the mention was posted" + ) + sixgill_mention_id = models.TextField( + unique=True, + blank=True, + null=True, + help_text="Cybersixgill mention ID associated with mention incident", + ) + post_id = models.TextField( + blank=True, + null=True, + help_text="Cybersixgill post ID associated with mention incident", + ) + lang = models.TextField( + blank=True, null=True, help_text="Language of the mention post" + ) + rep_grade = models.TextField( + blank=True, + null=True, + help_text="Threat actors reputation score determined by cyber sixgill", + ) + site = models.TextField( + blank=True, null=True, help_text="Site were the mention occured" + ) + site_grade = models.TextField( + blank=True, null=True, help_text="Grade of site where mention occured 0 - 5" + ) + title = models.TextField( + blank=True, null=True, help_text="Title of post where mention occured" + ) + type = models.TextField( + blank=True, null=True, help_text="Type of post where mention occured" + ) + url = models.TextField(blank=True, null=True, help_text="URL of mention post") + comments_count = models.TextField( + blank=True, null=True, help_text="Number of comments on the mention post" + ) + sub_category = models.TextField( + blank=True, null=True, help_text="Subcategory of mention" + ) + tags = models.TextField( + blank=True, null=True, help_text="Tags associated with mention alert" + ) + organization_uid = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organizations", + ) + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + title_translated = models.TextField( + blank=True, null=True, help_text="Title of mention post translated to english" + ) + content_translated = models.TextField( + blank=True, null=True, help_text="Content of mention post translated to english" + ) + detected_lang = models.TextField( + blank=True, null=True, help_text="Detected language of metion post" + ) + + class Meta: + """Set Mentions model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "mentions" + + +# Likely can be removed +class OrgIdMap(models.Model): + """Define OrgIdMap model.""" + + cyhy_id = models.TextField( + blank=True, null=True, help_text="Cyber Hygiene organization ID" + ) + pe_org_id = models.TextField( + blank=True, null=True, help_text="Posture & Exposure organization ID" + ) + merge_orgs = models.BooleanField( + blank=True, + null=True, + help_text="Boolean field to flag if the orgs should be merged", + ) + + class Meta: + """Set OrgIdMap model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "org_id_map" + unique_together = (("cyhy_id", "pe_org_id"),) + + +class OrgType(models.Model): + """Define OrgType model.""" + + org_type_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for organization type object.", + ) + org_type = models.TextField(blank=True, null=True, help_text="Organization type.") + + class Meta: + """Set OrgType model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "org_type" + + +# needs to be merged merged +# class Organizations(models.Model): +# """Define Organizations model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# name = models.TextField() +# cyhy_db_name = models.TextField(unique=True, blank=True, null=True) +# org_type_uid = models.ForeignKey( +# OrgType, +# on_delete=models.CASCADE, +# db_column="org_type_uid", +# blank=True, +# null=True, +# ) +# report_on = models.BooleanField(blank=True, null=True) +# password = models.TextField(blank=True, null=True) +# date_first_reported = models.DateTimeField(blank=True, null=True) +# parent_org_uid = models.ForeignKey( +# "self", +# on_delete=models.CASCADE, +# db_column="parent_org_uid", +# blank=True, +# null=True, +# ) +# premium_report = models.BooleanField(blank=True, null=True) +# agency_type = models.TextField(blank=True, null=True) +# demo = models.BooleanField(blank=True, null=True) +# scorecard = models.BooleanField(blank=True, null=True) +# fceb = models.BooleanField(blank=True, null=True) +# receives_cyhy_report = models.BooleanField(blank=True, null=True) +# receives_bod_report = models.BooleanField(blank=True, null=True) +# receives_cybex_report = models.BooleanField(blank=True, null=True) +# run_scans = models.BooleanField(blank=True, null=True) +# is_parent = models.BooleanField(blank=True, null=True) +# ignore_roll_up = models.BooleanField(blank=True, null=True) +# retired = models.BooleanField(blank=True, null=True) +# cyhy_period_start = models.DateField(blank=True, null=True) +# fceb_child = models.BooleanField(blank=True, null=True) +# election = models.BooleanField(blank=True, null=True) +# scorecard_child = models.BooleanField(blank=True, null=True) +# location_name = models.TextField(blank=True, null=True) +# county = models.TextField(blank=True, null=True) +# county_fips = models.IntegerField(blank=True, null=True) +# state_abbreviation = models.TextField(blank=True, null=True) +# state_fips = models.IntegerField(blank=True, null=True) +# state_name = models.TextField(blank=True, null=True) +# country = models.TextField(blank=True, null=True) +# country_name = models.TextField(blank=True, null=True) + +# class Meta: +# """Set Organizations model metadata.""" + +# managed = False +# db_table = "organizations" + + +class PshttResults(models.Model): + """Define PshttResults model.""" + + pshtt_results_uid = models.UUIDField( + primary_key=True, default=uuid.uuid1, help_text="" + ) + organization = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="", + ) + sub_domain = models.ForeignKey( + "SubDomains", on_delete=models.CASCADE, db_column="sub_domain_uid", help_text="" + ) + data_source = models.ForeignKey( + "DataSource", + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="", + ) + sub_domain = models.TextField(help_text="") + date_scanned = models.DateField(blank=True, null=True, help_text="") + base_domain = models.TextField(blank=True, null=True, help_text="") + base_domain_hsts_preloaded = models.BooleanField( + blank=True, null=True, help_text="" + ) + canonical_url = models.TextField(blank=True, null=True, help_text="") + defaults_to_https = models.BooleanField(blank=True, null=True, help_text="") + domain = models.TextField(blank=True, null=True, help_text="") + domain_enforces_https = models.BooleanField(blank=True, null=True, help_text="") + domain_supports_https = models.BooleanField(blank=True, null=True, help_text="") + domain_uses_strong_hsts = models.BooleanField(blank=True, null=True, help_text="") + downgrades_https = models.BooleanField(blank=True, null=True, help_text="") + htss = models.BooleanField(blank=True, null=True, help_text="") + hsts_entire_domain = models.BooleanField(blank=True, null=True, help_text="") + hsts_header = models.TextField(blank=True, null=True, help_text="") + hsts_max_age = models.DecimalField( + max_digits=1000, decimal_places=1000, blank=True, null=True, help_text="" + ) + hsts_preload_pending = models.BooleanField(blank=True, null=True, help_text="") + hsts_preload_ready = models.BooleanField(blank=True, null=True, help_text="") + hsts_preloaded = models.BooleanField(blank=True, null=True, help_text="") + https_bad_chain = models.BooleanField(blank=True, null=True, help_text="") + https_bad_hostname = models.BooleanField(blank=True, null=True, help_text="") + https_cert_chain_length = models.IntegerField(blank=True, null=True, help_text="") + https_client_auth_required = models.BooleanField( + blank=True, null=True, help_text="" + ) + https_custom_truststore_trusted = models.BooleanField( + blank=True, null=True, help_text="" + ) + https_expired_cert = models.BooleanField(blank=True, null=True, help_text="") + https_full_connection = models.BooleanField(blank=True, null=True, help_text="") + https_live = models.BooleanField(blank=True, null=True, help_text="") + https_probably_missing_intermediate_cert = models.BooleanField( + blank=True, null=True, help_text="" + ) + https_publicly_trusted = models.BooleanField(blank=True, null=True, help_text="") + https_self_signed_cert = models.BooleanField(blank=True, null=True, help_text="") + https_leaf_cert_expiration_date = models.DateField( + blank=True, null=True, help_text="" + ) + https_leaf_cert_issuer = models.TextField(blank=True, null=True, help_text="") + https_leaf_cert_subject = models.TextField(blank=True, null=True, help_text="") + https_root_cert_issuer = models.TextField(blank=True, null=True, help_text="") + ip = models.GenericIPAddressField(blank=True, null=True, help_text="") + live = models.BooleanField(blank=True, null=True, help_text="") + notes = models.TextField(blank=True, null=True, help_text="") + redirect = models.BooleanField(blank=True, null=True, help_text="") + redirect_to = models.TextField(blank=True, null=True, help_text="") + server_header = models.TextField(blank=True, null=True, help_text="") + server_version = models.TextField(blank=True, null=True, help_text="") + strictly_forces_https = models.BooleanField(blank=True, null=True, help_text="") + unknown_error = models.BooleanField(blank=True, null=True, help_text="") + valid_https = models.BooleanField(blank=True, null=True, help_text="") + ep_http_headers = models.TextField( + blank=True, null=True, help_text="" + ) # This field type is a guess. + ep_http_server_header = models.TextField(blank=True, null=True, help_text="") + ep_http_server_version = models.TextField(blank=True, null=True, help_text="") + ep_https_headers = models.TextField( + blank=True, null=True, help_text="" + ) # This field type is a guess. + ep_https_hsts_header = models.TextField(blank=True, null=True, help_text="") + ep_https_server_header = models.TextField(blank=True, null=True, help_text="") + ep_https_server_version = models.TextField(blank=True, null=True, help_text="") + ep_httpswww_headers = models.TextField( + blank=True, null=True, help_text="" + ) # This field type is a guess. + ep_httpswww_hsts_header = models.TextField(blank=True, null=True, help_text="") + ep_httpswww_server_header = models.TextField(blank=True, null=True, help_text="") + ep_httpswww_server_version = models.TextField(blank=True, null=True, help_text="") + ep_httpwww_headers = models.TextField( + blank=True, null=True, help_text="" + ) # This field type is a guess. + ep_httpwww_server_header = models.TextField(blank=True, null=True, help_text="") + ep_httpwww_server_version = models.TextField(blank=True, null=True, help_text="") + + class Meta: + """Set PshttResults model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "pshtt_results" + unique_together = (("organization", "sub_domain"),) + + +class PeReportSummaryStats(models.Model): + """Define ReportSummaryStats model.""" + + report_uid = models.UUIDField(primary_key=True, default=uuid.uuid1, help_text="") + organization = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="", + ) + start_date = models.DateField(help_text="") + end_date = models.DateField(blank=True, null=True, help_text="") + ip_count = models.IntegerField(blank=True, null=True, help_text="") + root_count = models.IntegerField(blank=True, null=True, help_text="") + sub_count = models.IntegerField(blank=True, null=True, help_text="") + ports_count = models.IntegerField(blank=True, null=True, help_text="") + creds_count = models.IntegerField(blank=True, null=True, help_text="") + breach_count = models.IntegerField(blank=True, null=True, help_text="") + cred_password_count = models.IntegerField(blank=True, null=True, help_text="") + domain_alert_count = models.IntegerField(blank=True, null=True, help_text="") + suspected_domain_count = models.IntegerField(blank=True, null=True, help_text="") + insecure_port_count = models.IntegerField(blank=True, null=True, help_text="") + verified_vuln_count = models.IntegerField(blank=True, null=True, help_text="") + suspected_vuln_count = models.IntegerField(blank=True, null=True, help_text="") + suspected_vuln_addrs_count = models.IntegerField( + blank=True, null=True, help_text="" + ) + threat_actor_count = models.IntegerField(blank=True, null=True, help_text="") + dark_web_alerts_count = models.IntegerField(blank=True, null=True, help_text="") + dark_web_mentions_count = models.IntegerField(blank=True, null=True, help_text="") + dark_web_executive_alerts_count = models.IntegerField( + blank=True, null=True, help_text="" + ) + dark_web_asset_alerts_count = models.IntegerField( + blank=True, null=True, help_text="" + ) + pe_number_score = models.TextField(blank=True, null=True, help_text="") + pe_letter_grade = models.TextField(blank=True, null=True, help_text="") + pe_percent_score = models.DecimalField( + max_digits=1000, decimal_places=1000, blank=True, null=True, help_text="" + ) + cidr_count = models.IntegerField(blank=True, null=True, help_text="") + port_protocol_count = models.IntegerField(blank=True, null=True, help_text="") + software_count = models.IntegerField(blank=True, null=True, help_text="") + foreign_ips_count = models.IntegerField(blank=True, null=True, help_text="") + + class Meta: + """Set ReportSummaryStats model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "pe_report_summary_stats" + unique_together = (("organization", "start_date"),) + + +class RootDomains(models.Model): + """Define RootDomains model.""" + + root_domain_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for root domains", + ) + organization = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organization", + ) + root_domain = models.TextField(help_text="Root domain") + ip_address = models.TextField( + blank=True, null=True, help_text="IP address of root domain" + ) + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + enumerate_subs = models.BooleanField( + blank=True, + null=True, + help_text="T/F should we identify subdomains for this root domain? (We don't enumerate for Cloud provider roots)", + ) + + class Meta: + """Set RootDomains model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "root_domains" + unique_together = (("root_domain", "organization"),) + + +class PeTeamMembers(models.Model): + """Define TeamMembers model.""" + + team_member_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a PE Team Member object.", + ) + team_member_fname = models.TextField( + blank=False, null=False, help_text="First name." + ) + team_member_lname = models.TextField(blank=False, null=False, help_text="Last Name") + team_member_email = models.TextField( + blank=False, null=False, help_text="Team member's email address." + ) + team_member_ghID = models.TextField( + blank=False, null=False, help_text="Team member's github ID." + ) + team_member_phone = models.TextField( + blank=True, null=True, help_text="Team member's phone number." + ) + team_member_role = models.TextField( + blank=True, null=True, help_text="Team member's role." + ) + team_member_notes = models.TextField( + blank=True, null=True, help_text="Notes about the team member." + ) + + class Meta: + """Set TeamMembers model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "pe_team_members" + + +class ShodanAssets(models.Model): + """Define ShodanAssets model.""" + + shodan_asset_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for shodan assets", + ) + organization = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + blank=True, + null=True, + help_text="FK: Foreign Key to organizations", + ) + # If you still need to store the organization name or acronym, use a separate field for that + organization_name = models.TextField( + blank=True, null=True, help_text="Organization name" + ) # New field to store the name or acronym + ip = models.TextField(blank=True, null=True, help_text="IP address") + port = models.IntegerField(blank=True, null=True, help_text="Port number") + protocol = models.TextField( + blank=True, null=True, help_text="Protocol running on the port" + ) + timestamp = models.DateTimeField( + blank=True, null=True, help_text="Time the asset was last seen by Shodan" + ) + product = models.TextField( + blank=True, null=True, help_text="What product is running on the asset" + ) + server = models.TextField( + blank=True, null=True, help_text="What server is running on the asset" + ) + tags = models.JSONField( + blank=True, + null=True, + help_text="shodan tags associated with the asset (ex. self-signed, vpn, starttls, cloud, etc.)", + ) # Store tags as a list (JSON format) + domains = models.JSONField( + blank=True, null=True, help_text="domains associated with the asset" + ) # Store domains as a list (JSON format) + hostnames = models.JSONField( + blank=True, null=True, help_text="hostnames associated with the asset" + ) # Store hostnames as a list (JSON format) + isp = models.TextField(blank=True, null=True, help_text="Internet service provider") + asn = models.IntegerField( + blank=True, null=True, help_text="Autonomous system number" + ) + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + blank=True, + null=True, + help_text="FK: Foreign Key to data_source", + ) + country_code = models.TextField( + blank=True, null=True, help_text="Country code where the IP was located." + ) + location = models.TextField( + blank=True, null=True, help_text="Location where the IP hosted." + ) + + class Meta: + """Set ShodanAssets model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "shodan_assets" + unique_together = (("organization", "ip", "port", "protocol", "timestamp"),) + + +# class ShodanInsecureProtocolsUnverifiedVulns(models.Model): +# """Define ShodanInsecureProtocolsUnverifiedVulns model.""" + +# insecure_product_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) +# organization_uid = models.ForeignKey( +# Organization, on_delete=models.CASCADE, db_column="organization_uid" +# ) +# organization = models.TextField(blank=True, null=True) +# ip = models.TextField(blank=True, null=True) +# port = models.IntegerField(blank=True, null=True) +# protocol = models.TextField(blank=True, null=True) +# type = models.TextField(blank=True, null=True) +# name = models.TextField(blank=True, null=True) +# potential_vulns = models.TextField( +# blank=True, null=True +# ) # This field type is a guess. +# mitigation = models.TextField(blank=True, null=True) +# timestamp = models.DateTimeField(blank=True, null=True) +# product = models.TextField(blank=True, null=True) +# server = models.TextField(blank=True, null=True) +# tags = models.TextField(blank=True, null=True) # This field type is a guess. +# domains = models.TextField(blank=True, null=True) # This field type is a guess. +# hostnames = models.TextField(blank=True, null=True) # This field type is a guess. +# isn = models.TextField(blank=True, null=True) +# asn = models.IntegerField(blank=True, null=True) +# data_source_uid = models.ForeignKey( +# DataSource, on_delete=models.CASCADE, db_column="data_source_uid" +# ) + +# class Meta: +# """Set ShodanInsecureProtocolsUnverifiedVulns model metadata.""" + +# managed = False +# db_table = "shodan_insecure_protocols_unverified_vulns" +# unique_together = ( +# ("organization_uid", "ip", "port", "protocol", "timestamp"), +# ) + + +class ShodanVulns(models.Model): + """Define ShodanVulns model.""" + + shodan_vuln_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a shodan vulnerability object.", + ) + organization = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organization", + ) + organization_name = models.TextField( + blank=True, null=True, help_text="Organization name" + ) + ip = models.TextField(blank=True, null=True, help_text="IP address") + port = models.TextField(blank=True, null=True, help_text="Port number") + protocol = models.TextField(blank=True, null=True, help_text="Protocol") + timestamp = models.DateTimeField( + blank=True, + null=True, + help_text="Timestamp when unverified vulnerability was found", + ) + cve = models.TextField( + blank=True, null=True, help_text="CVE associated with vulnerability" + ) + severity = models.TextField( + blank=True, + null=True, + help_text="Severity of vulnerability (medium, high, critical)", + ) + cvss = models.DecimalField( + max_digits=1000, + decimal_places=1000, + blank=True, + null=True, + help_text="Common Vulnerability Scoring System Score", + ) + summary = models.TextField( + blank=True, null=True, help_text="Summary of vulnerability" + ) + product = models.TextField( + blank=True, null=True, help_text="Product associated with vulnerability" + ) + attack_vector = models.TextField( + blank=True, null=True, help_text="Attack vector of vulnerability" + ) + av_description = models.TextField( + blank=True, null=True, help_text="Description of attack vector" + ) + attack_complexity = models.TextField( + blank=True, null=True, help_text="Complexity of attack (low, medium, high)" + ) + ac_description = models.TextField( + blank=True, null=True, help_text="Description of attack complexity" + ) + confidentiality_impact = models.TextField( + blank=True, + null=True, + help_text="Impact on confidentiality (none, partial, complete)", + ) + ci_description = models.TextField( + blank=True, null=True, help_text="Description of confidentiality impact" + ) + integrity_impact = models.TextField( + blank=True, null=True, help_text="Impact on integrity (none, partial complete)" + ) + ii_description = models.TextField( + blank=True, null=True, help_text="Description of integrity impact" + ) + availability_impact = models.TextField( + blank=True, + null=True, + help_text="Impact on availability (none, partial, complete)", + ) + ai_description = models.TextField( + blank=True, null=True, help_text="Description of availability impact" + ) + tags = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="List of tags associated with vulnerability", + ) # This field type is a guess. + domains = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="List of domains associated with vulnerability", + ) # This field type is a guess. + hostnames = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="Host names associated with vulnerability", + ) # This field type is a guess. + isp = models.TextField(blank=True, null=True, help_text="Internet service provider") + asn = models.IntegerField( + blank=True, null=True, help_text="Autonomous system number" + ) + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + null=True, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + type = models.TextField(blank=True, null=True, help_text="Type of vulnerability") + name = models.TextField(blank=True, null=True, help_text="Name of vulnerability") + potential_vulns = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="List of potential vulnerabilities associated with vulnerability", + ) # This field type is a guess. + mitigation = models.TextField( + blank=True, null=True, help_text="Information on how to mitigate vulnerability" + ) + server = models.TextField( + blank=True, null=True, help_text="Server associated with vulnerability" + ) + is_verified = models.BooleanField( + blank=True, null=True, help_text="T/F Is this a verified vulnerability?" + ) + banner = models.TextField( + blank=True, + null=True, + help_text="Snippet of information retrieved by Shodan about a service or device, typically revealing details like the software, version, and configuration of a system exposed to the internet.", + ) + version = models.TextField( + blank=True, null=True, help_text="Version of the server running.???" + ) + cpe = ArrayField( + models.TextField(blank=True, null=True), + blank=True, + null=True, + help_text="Common Platform Enumeration (CPE) id for the product the vulnerability was found on.", + ) + + class Meta: + """Set ShodanVulns model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "shodan_vulns" + unique_together = (("organization", "ip", "port", "protocol", "timestamp"),) + + +class SubDomains(models.Model): + """Define SubDomains model.""" + + sub_domain_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for subdomains", + ) + sub_domain = models.TextField( + help_text="Subdomain name" + ) # Crossfeed Domains name field + root_domain = models.ForeignKey( + RootDomains, + on_delete=models.CASCADE, + db_column="root_domain_uid", + help_text="FK: Foreign Key to root domains", + ) + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + dns_record = models.ForeignKey( + DnsRecords, + on_delete=models.CASCADE, + db_column="dns_record_uid", + blank=True, + null=True, + help_text="FK: Foreign Key to dns record", + ) + status = models.BooleanField( + blank=True, + null=True, + help_text="T/F: Boolean field flagging if the status is active.???", + ) + first_seen = models.DateField( + blank=True, + null=True, + help_text="Date and time of the first time teh subdomain was seen.", + ) + last_seen = models.DateField( + blank=True, null=True, help_text="Date of the last time the subdomain was seen." + ) + created_at = models.DateTimeField( + db_column="created_at", help_text="Datetime the subdomain object was created." + ) + updated_at = models.DateTimeField( + db_column="updated_at", help_text="Datetime the subdomain was last updated." + ) + current = models.BooleanField( + blank=True, + null=True, + help_text="T/F is this sub domain still live and linked to the organization", + ) + identified = models.BooleanField( + blank=True, + null=True, + help_text="T/F was this subdomain identified via an IP lookup.???", + ) + ip_address = models.TextField( + blank=True, null=True, help_text="IP address linked to the subdomain" + ) # XFD column + synced_at = models.DateTimeField( + db_column="synced_at", + blank=True, + null=True, + help_text="Date the subdomain was last synced", + ) # XFD column + from_root_domain = models.TextField( + db_column="from_root_domain", + blank=True, + null=True, + help_text="Root domain associated with the subdomain", + ) # XFD column + subdomain_source = models.TextField( + db_column="subdomain_source", + max_length=255, + blank=True, + null=True, + help_text="Where teh subdomain originated from.", + ) # XFD column + organization = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to organization", + ) + ip_only = models.BooleanField( + db_column="ip_only", + default=False, + help_text="T/F if there is no subdomain but just an IP", + ) # XFD column + reverse_name = models.CharField( + db_column="reverse_name", + max_length=512, + help_text="DNS reverse lookup of a subdomain", + ) # XFD column + screenshot = models.CharField( + max_length=512, + blank=True, + null=True, + help_text="link to the screenshot of the subdomain site.???", + ) # XFD Crossfeed Domains screenshot field + country = models.CharField( + max_length=255, + blank=True, + null=True, + help_text="Country where the subdomain is hosted.", + ) # XFD column + asn = models.CharField( + max_length=255, blank=True, null=True, help_text="Autonomous system number" + ) # XFD column + cloud_hosted = models.BooleanField( + db_column="cloud_hosted", + default=False, + help_text="T/F is this subdomain cloud hosted", + ) # XFD column + ssl = models.JSONField( + blank=True, + null=True, + help_text="SSL (Secure Sockets Layer) or TLS (Transport Layer Security) connection, certificate, or related security features.", + ) # XFD columnv + censys_certificates_results = models.JSONField( + db_column="censys_certificates_results", + default=dict, + help_text="Results from the censys certificate scan.", + ) # XFD column + trustymail_results = models.JSONField( + db_column="trustymail_results", + default=dict, + help_text="Results from the trustymail scan.", + ) # XFD column + + class Meta: + """Set SubDomains model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "sub_domains" + unique_together = (("sub_domain", "root_domain"),) + + +class TopCves(models.Model): + """Define TopCves model.""" + + top_cves_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: unique identifier for top cves", + ) + cve_id = models.TextField( + blank=True, null=True, help_text="CVE identifier ex. CVE-202-20038" + ) + dynamic_rating = models.TextField( + blank=True, + null=True, + help_text="CyberSixGill’s Dynamic Vulnerability Exploit (DVE) Score", + ) + nvd_base_score = models.TextField( + blank=True, + null=True, + help_text="Base CVE score from National vulnerability Databse", + ) + date = models.DateField( + blank=True, + null=True, + help_text="Date the CVE was fetched from the Cybersixgill API", + ) + summary = models.TextField(blank=True, null=True, help_text="Summary of the CVE") + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to data_source", + ) + + class Meta: + """Set TopCves model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "top_cves" + unique_together = (("cve_id", "date"),) + + +# Not sure if this is still used +class TopicTotals(models.Model): + """Define TopicTotals model.""" + + count_uuid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for topic_totals", + ) + organization_uid = models.UUIDField(help_text="FK: Foreign Key to organizations") + content_count = models.IntegerField( + help_text="Number dark web mentions that fit into a NLP topic" + ) + count_date = models.TextField( + blank=True, null=True, help_text="Date the count was taken" + ) + + class Meta: + """Set TopicTotals model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "topic_totals" + + +# Not sure if this is still used +class UniqueSoftware(models.Model): + """Define UniqueSoftware model.""" + + field_id = models.UUIDField( + db_column="_id", + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a unique software object.", + ) # Field renamed because it started with '_'. + software_name = models.TextField( + blank=False, null=False, help_text="Name of the software." + ) + + class Meta: + """Set UniqueSoftware model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "unique_software" + + +class WebAssets(models.Model): + """Define WebAssets model.""" + + asset_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifer for a web asset object.", + ) + asset_type = models.TextField(help_text="Type of web asset.") + asset = models.TextField(help_text="The web asset owned by the organization.") + ip_type = models.TextField( + blank=True, null=True, help_text="Type of IP if the asset is an IP" + ) + verified = models.BooleanField( + blank=True, null=True, help_text="T/F if the asset is verified or not." + ) + organization = models.ForeignKey( + Organization, + on_delete=models.CASCADE, + db_column="organization_uid", + help_text="FK: Foreign Key to the organization", + ) + asset_origin = models.TextField( + blank=True, null=True, help_text="Where the asset originated from." + ) + report_on = models.BooleanField( + blank=True, + null=True, + help_text="Whetherr or not PE should report on findings related to this asset.", + ) + last_scanned = models.DateTimeField( + blank=True, null=True, help_text="Last date the asset was scanned." + ) + report_status_reason = models.TextField( + blank=True, null=True, help_text="Reason the asset is not being reported on." + ) + data_source = models.ForeignKey( + DataSource, + on_delete=models.CASCADE, + db_column="data_source_uid", + help_text="FK: Foreign Key to the data source that created the web asset object.", + ) + + class Meta: + """Set WebAssets model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "web_assets" + unique_together = (("asset", "organization"),) + + +class WeeklyStatusesMdl(models.Model): + """Define WeeklyStatusesMdl model.""" + + weekly_status_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier to the weekly status object", + ) + user_status = models.TextField(blank=True, help_text="Name of the user.???") + key_accomplishments = models.TextField( + blank=True, + null=True, + help_text="Key accomplishments the user has made this week.", + ) + ongoing_task = models.TextField(help_text="Ongoing tasks the user is working on.") + upcoming_task = models.TextField( + help_text="Tasks the user has planned on starting." + ) + obstacles = models.TextField( + blank=True, + null=True, + help_text="Obstacles that the user is currently facing in accomplishing their tasks.", + ) + non_standard_meeting = models.TextField( + blank=True, + null=True, + help_text="Any non standard meetings during the last week.", + ) + deliverables = models.TextField( + blank=True, null=True, help_text="Key deliverables turned in by the user." + ) + pto = models.TextField(blank=True, null=True, help_text="Any upcoming PTO.") + week_ending = models.DateField(help_text="Last day of the week.") + notes = models.TextField(blank=True, null=True, help_text="additional notes") + statusComplete = models.IntegerField( + blank=True, + null=True, + help_text="T/F if the user has completed the status report.", + ) + + class Meta: + """Set WeeklyStatusesMdl model metadata.""" + + # unique_together = (('week_ending', 'user_status'),) + + app_label = app_label_name + managed = manage_db + db_table = "weekly_statuses_mdl" + + +# cyhy_kevs table model (needed for kev_list endpoint) +class CyhyKevs(models.Model): + """Define CyhyKevs model.""" + + cyhy_kevs_uid = models.UUIDField( + primary_key=True, help_text="PK: Unique identifier of the cyhy kev object." + ) + kev = models.CharField( + blank=True, null=True, max_length=255, help_text="CVE id of the KEV." + ) + first_seen = models.DateField( + blank=True, null=True, help_text="First time the KEV was seen." + ) + last_seen = models.DateField( + blank=True, null=True, help_text="Last time the KEV was seen." + ) + + class Meta: + """Set CyhyKevs model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "cyhy_kevs" + + +class XpanseBusinessUnits(models.Model): + """Define XpanseBusinessUnits model.""" + + xpanse_business_unit_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a xpanse business unit object.", + ) + entity_name = models.TextField( + unique=True, blank=True, null=True, help_text="Name of the business unit." + ) + cyhy_db_name = models.ForeignKey( + "Organization", + on_delete=models.CASCADE, + db_column="cyhy_db_name", + to_field="acronym", + null=True, # Allow NULL values + blank=True, + help_text="Acronym of the organization associated with the business unit.", + ) + state = models.TextField( + blank=True, null=True, help_text="State where the business unit is based." + ) + county = models.TextField( + blank=True, null=True, help_text="County where the business unit is based." + ) + city = models.TextField( + blank=True, null=True, help_text="City where the business unit is based." + ) + sector = models.TextField( + blank=True, null=True, help_text="Business unit's sector." + ) + entity_type = models.TextField( + blank=True, null=True, help_text="Type of business unit." + ) + region = models.TextField( + blank=True, null=True, help_text="Region where the business unit is based." + ) + rating = models.IntegerField( + blank=True, null=True, help_text="Xpanse rating of the business unit." + ) + + class Meta: + """Set XpanseBusinessUnits metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "xpanse_business_units" + + +class XpanseAssetsMdl(models.Model): + """Define XpanseAssetsMdl model.""" + + xpanse_asset_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for an Xpanse Asset object.", + ) + asm_id = models.TextField( + unique=True, blank=False, null=False, help_text="Xpanse ID for the asset" + ) + asset_name = models.TextField(blank=True, null=True, help_text="Name of the asset") + asset_type = models.TextField(blank=True, null=True, help_text="Type of asset") + last_observed = models.DateTimeField( + blank=True, null=True, help_text="Last datetime that the asset was observed" + ) + first_observed = models.DateTimeField( + blank=True, null=True, help_text="First datetime the asset was observed" + ) + externally_detected_providers = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of externally detected providers.", + ) + created = models.DateTimeField( + blank=True, null=True, help_text="Datetime the asset was created" + ) + ips = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of Ips associated with the asset.", + ) + active_external_services_types = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of active external services running on the asset.", + ) + domain = models.TextField( + blank=True, null=True, help_text="Domain associated with the asset." + ) + certificate_issuer = models.TextField( + blank=True, null=True, help_text="Certificate issuer." + ) + certificate_algorithm = models.TextField( + blank=True, null=True, help_text="Certificate algorithm" + ) + certificate_classifications = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of certificate classifications", + ) + resolves = models.BooleanField( + blank=True, null=True, help_text="T/F does the domain resolve to an live site." + ) + # details + top_level_asset_mapper_domain = models.TextField( + blank=True, null=True, help_text="The top level domain the subdomain maps to" + ) + domain_asset_type = models.JSONField( + blank=True, null=True, help_text="Type of the domain asset" + ) + is_paid_level_domain = models.BooleanField( + blank=True, null=True, help_text="T/F is the asset a paid level domain" + ) + domain_details = models.JSONField( + blank=True, null=True, help_text="Details about the domain" + ) + dns_zone = models.TextField( + blank=True, null=True, help_text="What zone does the dns resolve to." + ) + latest_sampled_ip = models.IntegerField( + blank=True, null=True, help_text="Latest IP seen on the domain" + ) + + recent_ips = models.JSONField( + blank=True, null=True, help_text="List of recent IPs linked to the domain" + ) + external_services = models.JSONField( + blank=True, null=True, help_text="External services running on the asset" + ) + externally_inferred_vulnerability_score = models.DecimalField( + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="Externally inferred vulnerability score", + ) + externally_inferred_cves = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Externally inferred CVEs", + ) + explainers = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Explainer text", + ) + tags = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Tags associated with the asset", + ) + + class Meta: + """Set XpanseAssetsMdl metdata.""" + + app_label = app_label_name + managed = manage_db + db_table = "xpanse_assets_mdl" + + +class XpanseCvesMdl(models.Model): + """Define XpanseCvesMdl model.""" + + xpanse_cve_uid = models.UUIDField( + unique=True, + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for an Xpanse CVE objct.", + ) + cve_id = models.TextField( + unique=True, blank=True, null=True, help_text="CVE identifier." + ) + cvss_score_v2 = models.DecimalField( + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="CVVS Score version 2", + ) + cve_severity_v2 = models.TextField( + blank=True, null=True, help_text="CVSS Severity Score version 2" + ) + cvss_score_v3 = models.DecimalField( + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="CVSS Score version 3", + ) + cve_severity_v3 = models.TextField( + blank=True, null=True, help_text="CVSS Severity Score version 3" + ) + + class Meta: + """Set XpanseCvesMdl metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "xpanse_cves_mdl" + + +class XpanseServicesMdl(models.Model): + """Define XpanseServicesMdl model.""" + + xpanse_service_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a Xpanse Service object.", + ) + service_id = models.TextField( + unique=True, + blank=True, + null=True, + help_text="Xpanse Identifier for the service.", + ) + service_name = models.TextField( + blank=True, null=True, help_text="Name of the service." + ) + service_type = models.TextField(blank=True, null=True, help_text="Type of service") + ip_address = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of IP addresses where the service is hosted, if applicable.", + ) + domain = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of domains where the service is hosted, if applicable.", + ) + externally_detected_providers = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of externally detected providers.", + ) + is_active = models.TextField( + blank=True, null=True, help_text="State of the service (Active, Inactive)." + ) + first_observed = models.DateTimeField( + blank=True, + null=True, + help_text="Datetime the service was first observed by Xpanse", + ) + last_observed = models.DateTimeField( + blank=True, + null=True, + help_text="Datetime the service was last observed by Xpanse", + ) + port = models.IntegerField( + blank=True, + null=True, + help_text="Number of the port where the service is running.", + ) + protocol = models.TextField( + blank=True, null=True, help_text="Protocol running on the port." + ) + active_classifications = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Current, actively detected and recognized software, technologies, or behaviors observed on a service based on the most recent data collected", + ) + inactive_classifications = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="previously detected and recognized software, technologies, or behaviors observed on a service previously, but not on the most recent data collected", + ) + discovery_type = models.TextField( + blank=True, null=True, help_text="How the service was detected." + ) + externally_inferred_vulnerability_score = models.DecimalField( + max_digits=5, + decimal_places=2, + blank=True, + null=True, + help_text="vulnerability score assigned to a service based on publicly available information about its product name and version, compared against known vulnerabilities in the National Vulnerability Database (NVD)", + ) + externally_inferred_cves = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="potential vulnerabilities identified by comparing the publicly visible version information of a service discovered on an organization's external attack surface with known vulnerabilities listed in the National Vulnerability Database (NVD)", + ) + service_key = models.TextField( + blank=True, + null=True, + help_text="identifier associated with a specific service that allows for access and interaction with that service within the Xpanse environment", + ) + service_key_type = models.TextField( + blank=True, null=True, help_text="Type of service key." + ) + + cves = models.ManyToManyField( + XpanseCvesMdl, + through="XpanseCveServiceMdl", + help_text="Many to many linking table to the cve table.", + ) + + class Meta: + """Set XpanseServicesMdl metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "xpanse_services_mdl" + + +class XpanseCveServiceMdl(models.Model): + """Define XpanseCves-Service linking table model.""" + + xpanse_inferred_cve = models.ForeignKey( + XpanseCvesMdl, + on_delete=models.CASCADE, + help_text="FK: Foreign key to the CVE associated with the service.", + ) + xpanse_service = models.ForeignKey( + XpanseServicesMdl, + on_delete=models.CASCADE, + help_text="FK: Foreign key to the service associated with the CVEs.", + ) + inferred_cve_match_type = models.TextField( + blank=True, + null=True, + help_text="If the match between service and CVE is approximate or exact.", + ) + product = models.TextField( + blank=True, + null=True, + help_text="Vulnerable product on the service that triggered the finding.", + ) + confidence = models.TextField( + blank=True, + null=True, + help_text="How confident Xpanse is the vulnerability is present.", + ) + vendor = models.TextField( + blank=True, null=True, help_text="Vendor who makes the compromised product." + ) + version_number = models.TextField( + blank=True, null=True, help_text="Version number of the compromised product." + ) + activity_status = models.TextField( + blank=True, + null=True, + help_text="Current activity status of the vulnerable product. (Inactive, Active)", + ) + first_observed = models.DateTimeField( + blank=True, + null=True, + help_text="First time the vulnerable product was seen running on the service.", + ) + last_observed = models.DateTimeField( + blank=True, + null=True, + help_text="Last time the vulnerable product was seen running on the service.", + ) + + class Meta: + """Set XpanseCveServiceMdl metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "xpanse_cve_services_mdl" + unique_together = (("xpanse_inferred_cve", "xpanse_service"),) + + +class XpanseAlerts(models.Model): + """Define XpanseAlerts model.""" + + xpanse_alert_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for an Xpanse alert object.", + ) + time_pulled_from_xpanse = models.DateTimeField( + blank=True, + null=True, + help_text="Time the alert was pulled from the Xpanse API.", + ) + alert_id = models.TextField( + unique=True, blank=False, null=False, help_text="Xpanse alert id." + ) + detection_timestamp = models.DateTimeField( + blank=True, null=True, help_text="Datetime the alert was detected by Xpanse." + ) + alert_name = models.TextField(blank=True, null=True, help_text="Name of the alert.") + # endpoint_id ???, + description = models.TextField( + blank=True, null=True, help_text="Description of the alert." + ) + host_name = models.TextField( + blank=True, null=True, help_text="IP or domain where the alert points." + ) + alert_action = models.TextField( + blank=True, + null=True, + help_text="a specific response or remediation step that is automatically taken when an alert is triggered.", + ) + # user_name ??? null, + # mac_addresses ??? null, + # source ??? null, + action_pretty = models.TextField( + blank=True, null=True, help_text="Human readable version of the alert action." + ) + # category ??? null, + # project ??? null, + # cloud_provider ??? null, + # resource_sub_type ??? null, + # resource_type ??? null, + action_country = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Country where the action was performed, if applicable.", + ) + # event_type ??? null, + # is_whitelisted ??? null, + # image_name ??? null, + # action_local_ip ??? null, + # action_local_port ??? null, + # action_external_hostname ??? null, + # action_remote_ip ??? null, + action_remote_port = ArrayField( + models.IntegerField(blank=True, null=False), + blank=True, + null=True, + help_text="Port number.", + ) + # "matching_service_rule_id ??? null, + starred = models.BooleanField( + blank=True, + null=True, + help_text="T/F if the user has starred the alert in the Xpanse system.", + ) + external_id = models.TextField( + blank=True, + null=True, + help_text="unique identifier that is used to reference a specific asset or record from the Xpanse system", + ) + related_external_id = models.TextField( + blank=True, + null=True, + help_text="Alert external id of the same alert being issued multiple times in the Xpanse system", + ) + alert_occurrence = models.IntegerField( + blank=True, + null=True, + help_text="Number of times the alert has been made in the Xpanse system for the same entity.", + ) + severity = models.TextField( + blank=True, null=True, help_text="Severity of the alert." + ) + matching_status = models.TextField( + blank=True, null=True, help_text="Status of the matching attempt." + ) + # end_match_attempt_ts ??? null, + local_insert_ts = models.DateTimeField( + blank=True, + null=True, + help_text="Datetime the alert was inserted into the mini data lake", + ) + last_modified_ts = models.DateTimeField( + blank=True, + null=True, + help_text="Datetime the alert was modified in the Xpanse system", + ) + case_id = models.IntegerField( + blank=True, null=True, help_text="Case id in the Xpanse system." + ) + # deduplicate_tokens ??? null, + # filter_rule_id ??? null, + # event_id ??? null, + event_timestamp = ArrayField( + models.DateTimeField(blank=True, null=False), + blank=True, + null=True, + help_text="List of event timestamps associated with the alert.", + ) + # action_local_ip_v6 ??? null, + # action_remote_ip_v6 ??? null, + alert_type = models.TextField(blank=True, null=True, help_text="Type of alert") + resolution_status = models.TextField( + blank=True, null=True, help_text="Current resolution status of the alert." + ) + resolution_comment = models.TextField( + blank=True, null=True, help_text="Comment about the resolution." + ) + # dynamic_fields ??? null, + tags = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of tags associated with the alert.", + ) + # malicious_urls ??? null, + last_observed = models.DateTimeField( + blank=True, null=True, help_text="Last time the issue was observed" + ) + country_codes = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="Country code associated with the alert", + ) + cloud_providers = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of cloud providers associated with the assets in the alert.", + ) + ipv4_addresses = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of IPs associated with the alert.", + ) + # ipv6_addresses ??? null, + domain_names = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of domains associated with the alert", + ) + service_ids = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of services ids associated with the alert", + ) + website_ids = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of websites associated with the alert.", + ) + asset_ids = ArrayField( + models.TextField(blank=True, null=False), + blank=True, + null=True, + help_text="List of asset's ids that are associated with the alert.", + ) + certificate = models.JSONField( + blank=True, + null=True, + help_text="Dictionary containing certificate data assocated with the alert.", + ) + # { + # issuerName": "IOS-Self-Signed-Certificate-782645061", + # subjectName": "IOS-Self-Signed-Certificate-782645061", + # validNotBefore": 1398850008000, + # validNotAfter": 1577836800000, + # serialNumber": "1" + # }, + port_protocol = models.TextField( + blank=True, null=True, help_text="Port protocol associated with the alert." + ) + # business_unit_hierarchies + attack_surface_rule_name = models.TextField( + blank=True, + null=True, + help_text="Attack surface rule that was triggered to create the alert", + ) + remediation_guidance = models.TextField( + blank=True, null=True, help_text="Guidance to remediate the alert." + ) + asset_identifiers = models.JSONField( + blank=True, + null=True, + help_text="List of dictionaries containg asset data associated with the alert", + ) + + business_units = models.ManyToManyField( + XpanseBusinessUnits, + related_name="alerts", + help_text="Many to many relationship to the related business units.", + ) + services = models.ManyToManyField( + XpanseServicesMdl, + related_name="alerts", + help_text="Many to many relationsthip to the services associated with the alert.", + ) + assets = models.ManyToManyField( + XpanseAssetsMdl, + related_name="alerts", + help_text="Many to many relationship to the assets associated with the alert.", + ) + + class Meta: + """Set XpanseAlerts model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "xpanse_alerts_mdl" + + +class CpeVender(models.Model): + """Define CpeVender model.""" + + cpe_vender_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique ID of the vender object", + ) + vender_name = models.TextField( + unique=True, blank=True, null=True, help_text="Vender name" + ) + + class Meta: + """Set CpeVender model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "cpe_vender" + + +class CpeProduct(models.Model): + """Define CpeProduct model.""" + + cpe_product_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for the Product (CPE)", + ) + cpe_product_name = models.TextField( + blank=True, null=True, help_text="Name of the product" + ) + version_number = models.TextField( + blank=True, null=True, help_text="Version of the product" + ) + cpe_vender = models.ForeignKey( + "CpeVender", + on_delete=models.CASCADE, + db_column="cpe_vender_uid", + default=None, + help_text="FK: Foreign key to the related vender object.", + ) + + # Create linking table for many to many relationship + cves = models.ManyToManyField( + Cve, + related_name="products", + help_text="Many to many relationship to the CVEs associated with the product", + ) + + class Meta: + """Set CpeProduct model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "cpe_product_mdl" + unique_together = (("cpe_product_name", "version_number"),) + + +# # THese are all views, so they shouldn't be generated via the ORM + +# # This should be a view not a table +# class VwPshttDomainsToRun(models.Model): +# """Define VwPshttDomainsToRun model.""" + +# sub_domain_uid = models.UUIDField(primary_key=True) +# sub_domain = models.TextField(blank=True, null=True) +# organization_uid = models.UUIDField() +# name = models.TextField(blank=True, null=True) + +# class Meta: +# """Set VwPshttDomainsToRun model metadata.""" + +# managed = False +# db_table = "vw_pshtt_domains_to_run" + + +# class VwBreachcompCredsbydate(models.Model): +# """Define VwBreachcompCredsbydate model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# mod_date = models.DateField(blank=True, null=True) +# no_password = models.BigIntegerField(blank=True, null=True) +# password_included = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwBreachcompCredsbydate model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_breachcomp_credsbydate" + + +# class VwDarkwebMentionsbydate(models.Model): +# """Define VwDarkwebMentionsbydate model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# count = models.BigIntegerField( +# db_column="Count", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebMentionsbydate model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_mentionsbydate" + + +# class VwShodanvulnsSuspected(models.Model): +# """Define VwShodanvulnsSuspected model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# organization = models.TextField(blank=True, null=True) +# ip = models.TextField(blank=True, null=True) +# port = models.TextField(blank=True, null=True) +# protocol = models.TextField(blank=True, null=True) +# type = models.TextField(blank=True, null=True) +# name = models.TextField(blank=True, null=True) +# potential_vulns = models.TextField( +# blank=True, null=True +# ) # This field type is a guess. +# mitigation = models.TextField(blank=True, null=True) +# timestamp = models.DateTimeField(blank=True, null=True) +# product = models.TextField(blank=True, null=True) +# server = models.TextField(blank=True, null=True) +# tags = models.TextField(blank=True, null=True) # This field type is a guess. +# domains = models.TextField(blank=True, null=True) # This field type is a guess. +# hostnames = models.TextField(blank=True, null=True) # This field type is a guess. +# isn = models.TextField(blank=True, null=True) +# asn = models.IntegerField(blank=True, null=True) +# data_source = models.TextField(blank=True, null=True) + +# class Meta: +# """Set VwShodanvulnsSuspected model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_shodanvulns_suspected" + + +# class VwShodanvulnsVerified(models.Model): +# """Define VwShodanvulnsVerified model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# organization = models.TextField(blank=True, null=True) +# ip = models.TextField(blank=True, null=True) +# port = models.TextField(blank=True, null=True) +# protocol = models.TextField(blank=True, null=True) +# timestamp = models.DateTimeField(blank=True, null=True) +# cve = models.TextField(blank=True, null=True) +# severity = models.TextField(blank=True, null=True) +# cvss = models.DecimalField( +# max_digits=1000, decimal_places=1000, blank=True, null=True +# ) +# summary = models.TextField(blank=True, null=True) +# product = models.TextField(blank=True, null=True) +# attack_vector = models.TextField(blank=True, null=True) +# av_description = models.TextField(blank=True, null=True) +# attack_complexity = models.TextField(blank=True, null=True) +# ac_description = models.TextField(blank=True, null=True) +# confidentiality_impact = models.TextField(blank=True, null=True) +# ci_description = models.TextField(blank=True, null=True) +# integrity_impact = models.TextField(blank=True, null=True) +# ii_description = models.TextField(blank=True, null=True) +# availability_impact = models.TextField(blank=True, null=True) +# ai_description = models.TextField(blank=True, null=True) +# tags = models.TextField(blank=True, null=True) # This field type is a guess. +# domains = models.TextField(blank=True, null=True) # This field type is a guess. +# hostnames = models.TextField(blank=True, null=True) # This field type is a guess. +# isn = models.TextField(blank=True, null=True) +# asn = models.IntegerField(blank=True, null=True) +# data_source = models.TextField(blank=True, null=True) +# banner = models.TextField(blank=True, null=True) +# version = models.TextField(blank=True, null=True) +# cpe = ArrayField( +# models.TextField(blank=True, null=True), blank=True, null=True +# ) + +# class Meta: +# """Set VwShodanvulnsVerified model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_shodanvulns_verified" + + +# class VwBreachcompBreachdetails(models.Model): +# """Define VwBreachcompBreachdetails model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# breach_name = models.TextField(blank=True, null=True) +# mod_date = models.DateField(blank=True, null=True) +# description = models.TextField(blank=True, null=True) +# breach_date = models.DateField(blank=True, null=True) +# password_included = models.BooleanField(blank=True, null=True) +# number_of_creds = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwBreachcompBreachdetails model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_breachcomp_breachdetails" + + +# class VwDarkwebSocmediaMostactposts(models.Model): +# """Define VwDarkwebSocmediaMostactposts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# title = models.TextField( +# db_column="Title", blank=True, null=True +# ) # Field name made lowercase. +# comments_count = models.IntegerField( +# db_column="Comments Count", blank=True, null=True +# ) # Field name made lowercase. Field renamed to remove unsuitable characters. + +# class Meta: +# """Set VwDarkwebSocmediaMostactposts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_socmedia_mostactposts" + + +# class VwDarkwebMostactposts(models.Model): +# """Define VwDarkwebMostactposts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# title = models.TextField( +# db_column="Title", blank=True, null=True +# ) # Field name made lowercase. +# comments_count = models.IntegerField( +# db_column="Comments Count", blank=True, null=True +# ) # Field name made lowercase. Field renamed to remove unsuitable characters. + +# class Meta: +# """Set VwDarkwebMostactposts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_mostactposts" + + +# class VwDarkwebAssetalerts(models.Model): +# """Define VwDarkwebAssetalerts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. +# title = models.TextField( +# db_column="Title", blank=True, null=True +# ) # Field name made lowercase. +# events = models.BigIntegerField( +# db_column="Events", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebAssetalerts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_assetalerts" + + +# class VwDarkwebExecalerts(models.Model): +# """Define VwDarkwebExecalerts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. +# title = models.TextField( +# db_column="Title", blank=True, null=True +# ) # Field name made lowercase. +# events = models.BigIntegerField( +# db_column="Events", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebExecalerts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_execalerts" + + +# class VwDarkwebThreatactors(models.Model): +# """Define VwDarkwebThreatactors model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# creator = models.TextField( +# db_column="Creator", blank=True, null=True +# ) # Field name made lowercase. +# grade = models.DecimalField( +# db_column="Grade", max_digits=1000, decimal_places=1000, blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebThreatactors model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_threatactors" + + +# class VwDarkwebPotentialthreats(models.Model): +# """Define VwDarkwebPotentialthreats model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. +# threats = models.TextField( +# db_column="Threats", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebPotentialthreats model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_potentialthreats" + + +# class VwDarkwebSites(models.Model): +# """Define VwDarkwebSites model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebSites model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_sites" + + +# class VwDarkwebInviteonlymarkets(models.Model): +# """Define VwDarkwebInviteonlymarkets model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# date = models.DateField(blank=True, null=True) +# site = models.TextField( +# db_column="Site", blank=True, null=True +# ) # Field name made lowercase. + +# class Meta: +# """Set VwDarkwebInviteonlymarkets model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_inviteonlymarkets" + + +# class VwDarkwebTopcves(models.Model): +# """Define VwDarkwebTopcves model.""" + +# top_cves_uid = models.UUIDField(primary_key=True) +# cve_id = models.TextField(blank=True, null=True) +# dynamic_rating = models.TextField(blank=True, null=True) +# nvd_base_score = models.TextField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# summary = models.TextField(blank=True, null=True) +# data_source_uid = models.UUIDField(blank=True, null=True) + +# class Meta: +# """Set VwDarkwebTopcves model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_darkweb_topcves" + + +# class VwCidrs(models.Model): +# """Define VwCidrs model.""" + +# cidr_uid = models.UUIDField(primary_key=True) +# network = models.TextField(blank=True, null=True) # This field type is a guess. +# organization_uid = models.UUIDField(blank=True, null=True) +# data_source_uid = models.UUIDField(blank=True, null=True) +# insert_alert = models.TextField(blank=True, null=True) + +# class Meta: +# """Set VwCidrs model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_cidrs" + + +# class VwBreachcomp(models.Model): +# """Define VwBreachcomp model.""" + +# credential_exposures_uid = models.UUIDField(primary_key=True) +# email = models.TextField(blank=True, null=True) +# breach_name = models.TextField(blank=True, null=True) +# organization_uid = models.UUIDField(blank=True, null=True) +# root_domain = models.TextField(blank=True, null=True) +# sub_domain = models.TextField(blank=True, null=True) +# hash_type = models.TextField(blank=True, null=True) +# name = models.TextField(blank=True, null=True) +# login_id = models.TextField(blank=True, null=True) +# password = models.TextField(blank=True, null=True) +# phone = models.TextField(blank=True, null=True) +# data_source_uid = models.UUIDField(blank=True, null=True) +# description = models.TextField(blank=True, null=True) +# breach_date = models.DateField(blank=True, null=True) +# added_date = models.DateTimeField(blank=True, null=True) +# modified_date = models.DateTimeField(blank=True, null=True) +# data_classes = models.TextField( +# blank=True, null=True +# ) # This field type is a guess. +# password_included = models.BooleanField(blank=True, null=True) +# is_verified = models.BooleanField(blank=True, null=True) +# is_fabricated = models.BooleanField(blank=True, null=True) +# is_sensitive = models.BooleanField(blank=True, null=True) +# is_retired = models.BooleanField(blank=True, null=True) +# is_spam_list = models.BooleanField(blank=True, null=True) + +# class Meta: +# """Set VwBreachcomp model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_breachcomp" + + +# class VwOrgsTotalDomains(models.Model): +# """Define VwOrgsTotalDomains model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# num_root_domain = models.BigIntegerField(blank=True, null=True) +# num_sub_domain = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsTotalDomains model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_total_domains" + + +# class VwOrgsContactInfo(models.Model): +# """Define VwOrgsContactInfo model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# agency_name = models.TextField(blank=True, null=True) +# contact_type = models.TextField(blank=True, null=True) +# contact_name = models.TextField(blank=True, null=True) +# email = models.TextField(blank=True, null=True) +# phone = models.TextField(blank=True, null=True) +# date_pulled = models.DateField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsContactInfo model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_contact_info" + + +# class VwOrgsTotalIps(models.Model): +# """Define VwOrgsTotalIps model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# num_ips = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsTotalIps model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_total_ips" + + +# class MatVwOrgsAllIps(models.Model): +# """Define MatVwOrgsAllIps model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# ip_addresses = ArrayField( +# models.GenericIPAddressField(blank=True, null=True), blank=True, null=True +# ) + +# class Meta: +# """Set MatVwOrgsAllIps model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "mat_vw_orgs_all_ips" + + +# class VwOrgsAttacksurface(models.Model): +# """Define VwOrgsAttacksurface model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# num_ports = models.BigIntegerField(blank=True, null=True) +# num_root_domain = models.BigIntegerField(blank=True, null=True) +# num_sub_domain = models.BigIntegerField(blank=True, null=True) +# num_ips = models.BigIntegerField(blank=True, null=True) +# num_cidrs = models.BigIntegerField(blank=True, null=True) +# num_ports_protocols = models.BigIntegerField(blank=True, null=True) +# num_software = models.BigIntegerField(blank=True, null=True) +# num_foreign_ips = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsAttacksurface model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_attacksurface" + + +# class VwOrgsTotalPorts(models.Model): +# """Define VwOrgsTotalPorts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.TextField(blank=True, null=True) +# num_ports = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwOrgsTotalPorts model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_orgs_total_ports" + + +# class VwIpsSubRootOrgInfo(models.Model): +# """VwIpsSubRootOrgInfo model class.""" + +# ip_hash = models.CharField(blank=True, null=True, max_length=255) +# ip = models.CharField(blank=True, null=True, max_length=255) +# origin_cidr = models.UUIDField(blank=True, null=True) +# organization_uid = models.UUIDField(blank=True, null=True) +# i_current = models.BooleanField(blank=True, null=True) +# sd_current = models.BooleanField(blank=True, null=True) + +# class Meta: +# """VwIpsSubRootOrgInfo model meta class.""" + +# managed = False +# db_table = "vw_ips_sub_root_org_info" + + +# class VwIpsCidrOrgInfo(models.Model): +# """VwIpsCidrOrgInfo model class.""" + +# ip_hash = models.CharField(blank=True, null=True, max_length=255) +# ip = models.CharField(blank=True, null=True, max_length=255) +# origin_cidr = models.UUIDField(blank=True, null=True) +# network = models.CharField(blank=True, null=True, max_length=255) +# organization_uid = models.UUIDField(blank=True, null=True) + +# class Meta: +# """VwIpsCidrOrgInfo model meta class.""" + +# managed = False +# db_table = "vw_ips_cidr_org_info" + + +# class VwPEScoreCheckNewCVE(models.Model): +# """VwPEScoreCheckNewCVE model class.""" + +# cve_name = models.CharField(blank=True, null=True, max_length=255) + +# class Meta: +# """VwPEScoreCheckNewCVE model meta class.""" + +# managed = False +# db_table = "vw_pescore_check_new_cve" + + +# # ---------- D-Score View Models ---------- +# # D-Score VS Cert View +# class VwDscoreVSCert(models.Model): +# """Define VwDscoreVSCert model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_ident_cert = models.BigIntegerField(blank=True, null=True) +# num_monitor_cert = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscoreVSCert model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_vs_cert" + + +# # D-Score VS Mail View +# class VwDscoreVSMail(models.Model): +# """Define VwDscoreVSMail model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_valid_dmarc = models.BigIntegerField(blank=True, null=True) +# num_valid_spf = models.BigIntegerField(blank=True, null=True) +# num_valid_dmarc_or_spf = models.BigIntegerField(blank=True, null=True) +# total_mail_domains = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscoreVSMail model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_vs_mail" + + +# # D-Score PE IP View +# class VwDscorePEIp(models.Model): +# """Define VwDscorePEIp model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_ident_ip = models.BigIntegerField(blank=True, null=True) +# num_monitor_ip = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscorePEIp model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_pe_ip" + + +# # D-Score PE Domain View +# class VwDscorePEDomain(models.Model): +# """Define VwDscorePEDomain model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_ident_domain = models.BigIntegerField(blank=True, null=True) +# num_monitor_domain = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscorePEDomain model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_pe_domain" + + +# # D-Score WAS Webapp View +# class VwDscoreWASWebapp(models.Model): +# """Define VwDscoreWASWebapp model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# num_ident_webapp = models.BigIntegerField(blank=True, null=True) +# num_monitor_webapp = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwDscoreWASWebapp model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_dscore_was_webapp" + + +# # ---------- I-Score View Models ---------- +# # I-Score VS Vuln View +# class VwIscoreVSVuln(models.Model): +# """Define VwIscoreVSVuln model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# cve_name = models.CharField(blank=True, null=True, max_length=255) +# cvss_score = models.FloatField(blank=True, null=True) + +# class Meta: +# """Set VwIscoreVSVuln model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_vs_vuln" + + +# # I-Score VS Vuln Previous View +# class VwIscoreVSVulnPrev(models.Model): +# """Define VwIscoreVSVulnPrev model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# cve_name = models.CharField(blank=True, null=True, max_length=255) +# cvss_score = models.FloatField(blank=True, null=True) +# time_closed = models.DateField(blank=True, null=True) + +# class Meta: +# """Set VwIscoreVSVulnPrev model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_vs_vuln_prev" + + +# # I-Score PE Vuln View +# class VwIscorePEVuln(models.Model): +# """Define VwIscorePEVuln model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# cve_name = models.CharField(blank=True, null=True, max_length=255) +# cvss_score = models.FloatField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePEVuln model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_vuln" + + +# # I-Score PE Cred View +# class VwIscorePECred(models.Model): +# """Define VwIscorePECred model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# password_creds = models.BigIntegerField(blank=True, null=True) +# total_creds = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePECred model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_cred" + + +# # I-Score PE Breach View +# class VwIscorePEBreach(models.Model): +# """Define VwIscorePEBreach model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# breach_count = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePEBreach model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_breach" + + +# # I-Score PE Darkweb View +# class VwIscorePEDarkweb(models.Model): +# """Define VwIscorePEDarkweb model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# alert_type = models.CharField(blank=True, null=True, max_length=255) +# date = models.DateField(blank=True, null=True) +# Count = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePEDarkweb model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_darkweb" + + +# # I-Score PE Protocol View +# class VwIscorePEProtocol(models.Model): +# """Define VwIscorePEProtocol model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# port = models.CharField(blank=True, null=True, max_length=255) +# ip = models.CharField(blank=True, null=True, max_length=255) +# protocol = models.CharField(blank=True, null=True, max_length=255) +# protocol_type = models.CharField(blank=True, null=True, max_length=255) +# date = models.DateField(blank=True, null=True) + +# class Meta: +# """Set VwIscorePEProtocol model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_pe_protocol" + + +# # I-Score WAS Vuln View +# class VwIscoreWASVuln(models.Model): +# """Define VwIscoreWASVuln model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) +# cve_name = models.CharField(blank=True, null=True, max_length=255) +# cvss_score = models.FloatField(blank=True, null=True) +# owasp_category = models.CharField(blank=True, null=True, max_length=255) + +# class Meta: +# """Set VwIscoreWASVuln model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_was_vuln" + + +# # I-Score WAS Vuln Previous View +# class VwIscoreWASVulnPrev(models.Model): +# """Define VwIscoreWASVulnPrev model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# parent_org_uid = models.UUIDField(blank=True, null=True) +# was_total_vulns_prev = models.BigIntegerField(blank=True, null=True) +# date = models.DateField(blank=True, null=True) + +# class Meta: +# """Set VwIscoreWASVulnPrev model metadata.""" + +# managed = False # Created from a view. Don't remove. +# db_table = "vw_iscore_was_vuln_prev" + + +# # ---------- Misc. Score Related Models ---------- +# # vw_iscore_orgs_ip_counts view model (used for XS/S/M/L/XL orgs endpoints) +# class VwIscoreOrgsIpCounts(models.Model): +# """Define VwIscoreOrgsIpCounts model.""" + +# organization_uid = models.UUIDField(primary_key=True) +# cyhy_db_name = models.CharField(blank=True, null=True, max_length=255) +# ip_count = models.BigIntegerField(blank=True, null=True) + +# class Meta: +# """Set VwIscoreOrgsIpCounts model metadata.""" + +# managed = False +# db_table = "vw_iscore_orgs_ip_counts"""" Django ORM models """ diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/tests.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/tests.py new file mode 100644 index 00000000..7ce503c2 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/pe_reports/pe_reports_django_project/dmz_mini_dl/views.py b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/views.py new file mode 100644 index 00000000..91ea44a2 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/dmz_mini_dl/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/src/pe_reports/pe_reports_django_project/home/models.py b/src/pe_reports/pe_reports_django_project/home/models.py index e97094fb..647ddc6e 100644 --- a/src/pe_reports/pe_reports_django_project/home/models.py +++ b/src/pe_reports/pe_reports_django_project/home/models.py @@ -731,7 +731,7 @@ class Meta: """Set IpsSubs model metadata.""" managed = False - # db_table = 'ips_subs' + db_table = "ips_subs" unique_together = (("ip_hash", "sub_domain_uid"),) @@ -1049,7 +1049,7 @@ class Meta: class ShodanAssets(models.Model): """Define ShodanAssets model.""" - shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + shodan_asset_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organizations_uid = models.ForeignKey( Organizations, on_delete=models.CASCADE, db_column="organizations_uid" ) @@ -1060,9 +1060,11 @@ class ShodanAssets(models.Model): timestamp = models.DateTimeField(blank=True, null=True) product = models.TextField(blank=True, null=True) server = models.TextField(blank=True, null=True) - tags = models.TextField(blank=True, null=True) # This field type is a guess. - domains = models.TextField(blank=True, null=True) # This field type is a guess. - hostnames = models.TextField(blank=True, null=True) # This field type is a guess. + tags = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + domains = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + hostnames = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) isn = models.TextField(blank=True, null=True) asn = models.IntegerField(blank=True, null=True) data_source_uid = models.ForeignKey( @@ -1084,7 +1086,7 @@ class Meta: class ShodanInsecureProtocolsUnverifiedVulns(models.Model): """Define ShodanInsecureProtocolsUnverifiedVulns model.""" - insecure_product_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + insecure_product_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organizations_uid = models.ForeignKey( Organizations, on_delete=models.CASCADE, db_column="organizations_uid" ) @@ -1123,7 +1125,7 @@ class Meta: class ShodanVulns(models.Model): """Define ShodanVulns model.""" - shodan_vuln_uid = models.UUIDField(primary_key=True, default=uuid.uuid1()) + shodan_vuln_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) organizations_uid = models.ForeignKey( Organizations, on_delete=models.CASCADE, db_column="organizations_uid" ) @@ -1149,9 +1151,11 @@ class ShodanVulns(models.Model): ii_description = models.TextField(blank=True, null=True) availability_impact = models.TextField(blank=True, null=True) ai_description = models.TextField(blank=True, null=True) - tags = models.TextField(blank=True, null=True) # This field type is a guess. - domains = models.TextField(blank=True, null=True) # This field type is a guess. - hostnames = models.TextField(blank=True, null=True) # This field type is a guess. + tags = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + domains = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) + hostnames = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) isn = models.TextField(blank=True, null=True) asn = models.IntegerField(blank=True, null=True) data_source_uid = models.ForeignKey( @@ -1159,18 +1163,16 @@ class ShodanVulns(models.Model): ) type = models.TextField(blank=True, null=True) name = models.TextField(blank=True, null=True) - potential_vulns = models.TextField( - blank=True, null=True - ) # This field type is a guess. + potential_vulns = ArrayField( + models.TextField(blank=True, null=True), blank=True, null=True + ) mitigation = models.TextField(blank=True, null=True) server = models.TextField(blank=True, null=True) is_verified = models.BooleanField(blank=True, null=True) banner = models.TextField(blank=True, null=True) version = models.TextField(blank=True, null=True) mitigation = models.TextField(blank=True, null=True) - cpe = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True - ) + cpe = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) class Meta: """Set ShodanVulns model metadata.""" @@ -1447,9 +1449,7 @@ class VwShodanvulnsVerified(models.Model): data_source = models.TextField(blank=True, null=True) banner = models.TextField(blank=True, null=True) version = models.TextField(blank=True, null=True) - cpe = ArrayField( - models.TextField(blank=True, null=True), blank=True, null=True - ) + cpe = ArrayField(models.TextField(blank=True, null=True), blank=True, null=True) class Meta: """Set VwShodanvulnsVerified model metadata.""" @@ -2116,6 +2116,12 @@ class XpanseBusinessUnits(models.Model): xpanse_business_unit_uid = models.UUIDField(primary_key=True, default=uuid.uuid1) entity_name = models.TextField(unique=True, blank=True, null=True) + cyhy_db_name = models.ForeignKey( + "Organizations", + on_delete=models.CASCADE, + db_column="cyhy_db_name", + to_field="cyhy_db_name", + ) state = models.TextField(blank=True, null=True) county = models.TextField(blank=True, null=True) city = models.TextField(blank=True, null=True) @@ -2464,28 +2470,30 @@ class Meta: """ -- WARNING: It may differ from actual native database DDL CREATE TABLE information_schema.was_findings ( - finding_uid uuid NOT NULL, - finding_type varchar(10485760) NULL, - webapp_id int4 NULL, - was_org_id text NULL, - owasp_category varchar(10485760) NULL, - severity varchar(10485760) NULL, - times_detected int4 NULL, - base_score float8 NULL, - temporal_score float8 NULL, - fstatus varchar(10485760) NULL, - last_detected date NULL, - first_detected date NULL, - is_remidiated bool NULL, - potential bool NULL, - webapp_url text NULL, - webapp_name text NULL, - "name" text NULL, - cvss_v3_attack_vector text NULL, - cwe_list _int4 NULL, - wasc_list jsonb NULL + finding_uid uuid NOT NULL, + finding_type varchar(10485760) NULL, + webapp_id int4 NULL, + was_org_id text NULL, + owasp_category varchar(10485760) NULL, + severity varchar(10485760) NULL, + times_detected int4 NULL, + base_score float8 NULL, + temporal_score float8 NULL, + fstatus varchar(10485760) NULL, + last_detected date NULL, + first_detected date NULL, + is_remidiated bool NULL, + potential bool NULL, + webapp_url text NULL, + webapp_name text NULL, + "name" text NULL, + cvss_v3_attack_vector text NULL, + cwe_list _int4 NULL, + wasc_list jsonb NULL ); """ + + class WasFindings(models.Model): """Define WasFindings model.""" @@ -2511,7 +2519,59 @@ class WasFindings(models.Model): models.IntegerField(blank=True, null=True), blank=True, null=True ) wasc_list = models.JSONField(blank=True, null=True) + class Meta: """Set WasFindings model metadata.""" + managed = False db_table = "was_findings" + +class WasReport(models.Model): + org_name = models.TextField(blank=True, null=True) + date_pulled = models.DateTimeField(blank=True, null=True) + last_scan_date = models.DateTimeField(blank=True, null=True) + security_risk = models.TextField(blank=True, null=True) + total_info = models.IntegerField(blank=True, null=True) + num_apps = models.IntegerField(blank=True, null=True) + risk_color = models.TextField(blank=True, null=True) + sensitive_count = models.IntegerField(blank=True, null=True) + sensitive_color = models.TextField(blank=True, null=True) + max_days_open_urgent = models.IntegerField(blank=True, null=True) + max_days_open_critical = models.IntegerField(blank=True, null=True) + urgent_color = models.TextField(blank=True, null=True) + critical_color = models.TextField(blank=True, null=True) + org_was_acronym = models.TextField(blank=True, null=True) + name_len = models.TextField(blank=True, null=True) + vuln_csv_dict = models.JSONField(blank=True, null=True, default=dict) + ssn_cc_dict = models.JSONField(blank=True, null=True, default=dict) + app_overview_csv_dict = models.JSONField(blank=True, null=True, default=dict) + details_csv = models.JSONField(blank=True, null=True, default=list) + info_csv = models.JSONField(blank=True, null=True, default=list) + links_crawled = models.JSONField(blank=True, null=True, default=list) + links_rejected = models.JSONField(blank=True, null=True, default=list) + emails_found = models.JSONField(blank=True, null=True, default=list) + owasp_count_dict = models.JSONField(blank=True, null=True, default=dict) + group_count_dict = models.JSONField(blank=True, null=True, default=dict) + fixed = models.IntegerField(blank=True, null=True) + total = models.IntegerField(blank=True, null=True) + vulns_monthly_dict = models.JSONField(blank=True, null=True, default=dict) + path_disc = models.IntegerField(blank=True, null=True) + info_disc = models.IntegerField(blank=True, null=True) + cross_site = models.IntegerField(blank=True, null=True) + burp = models.IntegerField(blank=True, null=True) + sql_inj = models.IntegerField(blank=True, null=True) + bugcrowd = models.IntegerField(blank=True, null=True) + reopened = models.IntegerField(blank=True, null=True) + reopened_color = models.TextField(blank=True, null=True) + new_vulns = models.IntegerField(blank=True, null=True) + new_vulns_color = models.TextField(blank=True, null=True) + tot_vulns = models.IntegerField(blank=True, null=True) + tot_vulns_color = models.TextField(blank=True, null=True) + lev1 = models.IntegerField(blank=True, null=True) + lev2 = models.IntegerField(blank=True, null=True) + lev3 = models.IntegerField(blank=True, null=True) + lev4 = models.IntegerField(blank=True, null=True) + lev5 = models.IntegerField(blank=True, null=True) + severities = ArrayField(models.IntegerField(), blank=True, null=True, default=list) + ages = ArrayField(models.IntegerField(), blank=True, null=True, default=list) + pdf_obj = models.BinaryField(blank=True, null=True) diff --git a/src/pe_reports/pe_reports_django_project/home/views.py b/src/pe_reports/pe_reports_django_project/home/views.py index ea85a437..e86a5dde 100644 --- a/src/pe_reports/pe_reports_django_project/home/views.py +++ b/src/pe_reports/pe_reports_django_project/home/views.py @@ -76,7 +76,7 @@ def getUserKey(): """Get a users API key.""" - urlIDs = "http://127.0.0.1:8089/apiv1/get_key" + urlIDs = "http://127.0.0.1:8091/apiv1/get_key" payload = json.dumps({"refresh_token": f'{config("USER_REFRESH_TOKEN")}'}) headers = { "Content-Type": "application/json", diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py b/src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py old mode 100644 new mode 100755 index 0ca7fdf9..5e57a42a --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/asgi.py @@ -8,9 +8,9 @@ """ # Standard Python Libraries import os - -# Third-Party Libraries -from dataAPI.views import api_router +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pe_reports_django.settings") +import django +django.setup() # Following 2 lines custom code from django.apps import apps @@ -21,9 +21,10 @@ from fastapi.staticfiles import StaticFiles from starlette.middleware.cors import CORSMiddleware -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pe_reports_django.settings") +# Third-Party Libraries +from dataAPI.views import api_router -application = get_wsgi_application() +# application = get_wsgi_application() # Below this comment is custom code apps.populate(settings.INSTALLED_APPS) diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/db_routers.py b/src/pe_reports/pe_reports_django_project/pe_reports_django/db_routers.py new file mode 100755 index 00000000..e0a9bc66 --- /dev/null +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/db_routers.py @@ -0,0 +1,32 @@ +class MyAppRouter: + def db_for_read(self, model, **hints): + # Specify the app you want to route to the mini_data_lake database + if model._meta.app_label == 'dmz_mini_dl': + return 'mini_data_lake' + return 'default' # All other models go to the default database + + def db_for_write(self, model, **hints): + if model._meta.app_label == 'dmz_mini_dl': + return 'mini_data_lake' + return 'default' # All other models go to the default database + + def allow_relation(self, obj1, obj2, **hints): + # Check the app labels of both objects + app_label1 = obj1._meta.app_label + app_label2 = obj2._meta.app_label + + # If both objects are from the specific app, allow the relation + if app_label1 == 'dmz_mini_dl' and app_label2 == 'dmz_mini_dl': + return True + + # If only one of them is from the specific app, disallow the relation + if app_label1 == 'dmz_mini_dl' or app_label2 == 'dmz_mini_dl': + return False + + # Allow relations between all other models + return True + + def allow_migrate(self, db, app_label, model_name=None, **hints): + if app_label == 'dmz_mini_dl': + return db == 'mini_data_lake' # Migrate the specific app to the mini_data_lake database + return db == 'default' # All other apps migrate to the default database \ No newline at end of file diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt b/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt index 0671ea45..55ebd005 100644 --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/requirements.txt @@ -1,254 +1,23 @@ -alembic==1.8.1 -amqp==5.1.1 -anyio==3.6.1 -arabic-reshaper==2.1.3 -asgiref==3.6.0 -asn1crypto==1.5.1 -asttokens==2.0.5 -async-timeout==4.0.2 -attrs==21.4.0 -Babel==2.12.1 -backcall==0.2.0 -bcrypt==4.0.1 -beautifulsoup4==4.10.0 -billiard==3.6.4.0 -blis==0.7.8 -boto3==1.21.10 -botocore==1.24.10 -build==0.7.0 -catalogue==2.0.6 -celery==5.2.7 -certifi==2021.10.8 -cffi==1.15.0 -cfgv==3.3.1 -channels==4.0.0 -chardet==3.0.4 -charset-normalizer==2.0.12 -check-manifest==0.48 -chevron==0.14.0 -circlify==0.15.0 -click==8.1.3 -click-didyoumean==0.3.0 -click-plugins==1.1.1 -click-repl==0.2.0 -colorama==0.4.4 -contextlib2==21.6.0 -coverage==6.3.2 -coveralls==3.3.1 -crispy-bootstrap5==0.7 -cryptography==36.0.2 -cssselect2==0.7.0 -cycler==0.11.0 -cymem==2.0.6 -DataProperty==0.55.0 -dateparser==1.1.0 -decorator==5.1.1 -demoji==1.1.0 -Deprecated==1.2.13 -distlib==0.3.4 -Django==4.1.5 -django-crispy-forms==1.14.0 -django-database-view==0.3.0 -dnspython==2.2.1 -dnstwist==20220815 -docopt==0.6.2 -docx==0.2.4 -docxcompose==1.4.0 -docxtpl==0.16.6 -dshield==0.2.1 -ecdsa==0.18.0 -email-validator==1.3.0 -en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.4.0/en_core_web_lg-3.4.0-py3-none-any.whl -en-core-web-trf @ https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.4.0/en_core_web_trf-3.4.0-py3-none-any.whl -et-xmlfile==1.1.0 -executing==0.8.3 -Faker==13.3.2 -fastapi==0.88.0 -filelock==3.6.0 -Flask==2.1.2 -Flask-Login==0.6.1 -Flask-Migrate==3.1.0 -Flask-SQLAlchemy==2.5.1 -Flask-WTF==1.0.1 -flower==1.2.0 -future==0.18.2 -glob2==0.7 -googletrans==4.0.0rc1 -greenlet==2.0.0a2 -gunicorn==20.1.0 -h11==0.9.0 -h2==3.2.0 -hpack==3.0.0 -hstspreload==2021.12.1 -html5lib==1.1 -httpcore==0.9.1 -httptools==0.5.0 -httpx==0.13.3 -huggingface-hub==0.8.1 -humanize==4.6.0 -hyperframe==5.2.0 -identify==2.5.0 -idna==2.5 -importlib-resources==5.4.0 -iniconfig==1.1.1 -ipython==8.4.0 -itsdangerous==2.1.2 -jedi==0.18.1 -Jinja2==3.1.2 -jmespath==0.10.0 -joblib==1.1.0 -jsonschema==4.4.0 -kiwisolver==1.3.2 -kombu==5.2.4 -langcodes==3.3.0 -limits==3.4.0 -lxml==4.8.0 -Mako==1.2.1 -MarkupSafe==2.1.1 -matplotlib==3.3.4 -matplotlib-inline==0.1.3 -mbstrdecoder==1.1.0 -mongo-db-from-config @ http://github.com/cisagov/mongo-db-from-config/tarball/develop -murmurhash==1.0.7 -mypy==0.961 -mypy-extensions==0.4.3 -nassl==4.0.2 -nltk==3.7 -nodeenv==1.6.0 -numpy==1.22.3 -openpyxl==3.0.9 -oscrypto==1.3.0 -packaging==21.3 -pandas==1.1.5 -paramiko==2.12.0 -parso==0.8.3 -path==16.4.0 -pathvalidate==2.5.0 -pathy==0.6.2 -pdfkit==1.0.0 --e git+https://github.com/cisagov/pe-reports.git@eda9a332132581c12b8620ed925e4b82d0d62316#egg=pe_reports -pep517==0.12.0 -pexpect==4.8.0 -phonenumbers==8.12.45 -pickleshare==0.7.5 -Pillow==9.0.1 -platformdirs==2.5.1 -pluggy==1.0.0 -pre-commit==2.18.1 -preshed==3.0.6 -prometheus-client==0.16.0 -prompt-toolkit==3.0.38 --e git+https://github.com/cisagov/pshtt.git@65596ff08fa7bf0357b5af63da73e2dead91c304#egg=pshtt -psutil==5.9.1 -psycopg2-binary==2.9.3 -ptyprocess==0.7.0 -publicsuffix==1.1.1 -pure-eval==0.2.2 -py==1.11.0 -pyasn1==0.4.8 -pycparser==2.21 -pydantic==1.9.0 -Pygments==2.12.0 -pygtail==0.12.0 -pyHanko==0.16.0 -pyhanko-certvalidator==0.19.8 -pymongo==4.0.1 -PyMuPDF==1.19.0 -PyNaCl==1.5.0 -pyOpenSSL==22.0.0 -pyparsing==3.0.7 -PyPDF2==1.26.0 -PyPDF3==1.0.6 -pyrsistent==0.18.1 -pytablereader==0.31.3 -pytablewriter==0.64.2 -pytest==7.0.1 -pytest-cov==3.0.0 -python-bidi==0.4.2 -python-dateutil==2.8.2 -python-decouple==3.6 -python-docx==0.8.11 -python-dotenv==0.21.0 -python-jose==3.3.0 -python-multipart==0.0.5 -python-pptx==0.6.21 -python-stdnum==1.17 -pytz==2023.3 -pytz-deprecation-shim==0.1.0.post0 -PyYAML==6.0 -qrcode==7.3.1 -rabbitmq==0.2.0 -regex==2022.3.15 -reportlab==3.6.6 -requests==2.27.1 -retry==0.9.2 -rfc3986==1.5.0 -rsa==4.9 -s3transfer==0.5.2 -schema==0.7.5 -scikit-learn==1.0.2 -scipy==1.8.0 -scrubadub==2.0.0 -semver==2.13.0 -shodan==1.27.0 -six==1.16.0 -sklearn==0.0 -slowapi==0.1.8 -smart-open==5.2.1 -sniffio==1.2.0 -soupsieve==2.3.1 -spacy==3.4.0 -spacy-alignments==0.8.5 -spacy-legacy==3.0.9 -spacy-loggers==1.0.3 -spacy-transformers==1.1.7 -SQLAlchemy==1.4.39 -sqlparse==0.4.3 -srsly==2.4.4 -sshtunnel==0.4.0 -sslyze==5.0.3 -stack-data==0.3.0 -starlette==0.22.0 -Sublist3r==1.0 -svglib==1.5.1 -tabledata==1.3.0 -tcolorpy==0.1.2 -textblob==0.15.3 -thinc==8.1.0 -threadpoolctl==3.1.0 -tinycss2==1.2.1 -tls-parser==2.0.0 -tokenizers==0.12.1 -toml==0.10.2 -tomli==2.0.1 -torch==1.12.0 -tornado==6.3.1 -tox==3.24.5 -tqdm==4.63.0 -traitlets==5.3.0 -transformers==4.20.1 -typepy==1.3.0 -typer==0.4.2 -types-PyYAML==6.0.4 -typing_extensions==4.1.1 -tzdata==2021.5 -tzlocal==4.1 -unicorn==2.0.1.post1 -uritools==4.0.1 -urllib3==1.26.0 -uvicorn==0.18.3 -uvloop==0.17.0 -vine==5.0.0 -virtualenv==20.14.0 -wasabi==0.9.1 -watchfiles==0.17.0 -wcwidth==0.2.6 -webencodings==0.5.1 -websockets==10.3 -Werkzeug==2.0.0 -wget==3.2 -whitenoise==6.3.0 -wrapt==1.15.0 -WTForms==3.0.1 -xhtml2pdf==0.2.5 -XlsxWriter==3.0.3 +crispy-bootstrap5 +Django +django-celery-beat +django-crispy-forms +django-netfields +docxtpl +elastic-apm +email_validator +en-core-web-lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.5.0/en_core_web_lg-3.5.0-py3-none-any.whl#sha256=c8ac64840c1eb3e3ca7bd38bd1e1c48fb0faeb2449d54d01d5ce629af4595775 +fastapi +fastapi-limiter +flower +numpy==1.23.5 +-e git+https://github.com/cisagov/ATC-Framework.git@CD-add-CODEOWNERS#egg=pe_reports +psycopg2-binary +python-decouple==3.8 +python-jose +python-multipart +rabbitmq +retry +slowapi +uvicorn +whitenoise diff --git a/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py b/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py old mode 100644 new mode 100755 index cedc5a7f..27f82bfc --- a/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py +++ b/src/pe_reports/pe_reports_django_project/pe_reports_django/settings.py @@ -57,6 +57,7 @@ "django.contrib.messages", "django.contrib.staticfiles", "dataAPI.apps.DataapiConfig", + "dmz_mini_dl.apps.DmzMiniDlConfig", "bulkupload.apps.BulkuploadConfig", "home.apps.HomeConfig", "manage_login.apps.ManageLoginConfig", @@ -99,7 +100,7 @@ "class": "logging.handlers.RotatingFileHandler", "maxBytes": 1024 * 1024 * 15, "backupCount": 10, - "filename": "./pe_reportsLogFile.log", + "filename": os.path.join(BASE_DIR, "pe_reportsLogFile.log"), "formatter": "verbose", }, }, @@ -178,9 +179,19 @@ "PASSWORD": config("password"), "HOST": config("host"), "PORT": config("port"), + }, + "mini_data_lake": { + 'ENGINE': "django.db.backends.postgresql_psycopg2", # Replace with your database engine + 'NAME': config("mdl_database"), + 'USER': config('mdl_user'), + 'PASSWORD': config('mdl_password'), + 'HOST': config('mdl_host'), + 'PORT': config('mdl_port'), } } +DATABASE_ROUTERS = ['pe_reports_django.db_routers.MyAppRouter'] + # Celery settings CELERY_BROKER_URL = ( f"amqp://{config('RABBITMQ_USER')}:{config('RABBITMQ_PASS')}@localhost:5672/" diff --git a/src/pe_reports/pe_reports_django_project/static/CISAImage.png b/src/pe_reports/pe_reports_django_project/static/CISAImage.png old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/autocomplete.css b/src/pe_reports/pe_reports_django_project/static/admin/css/autocomplete.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/base.css b/src/pe_reports/pe_reports_django_project/static/admin/css/base.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/changelists.css b/src/pe_reports/pe_reports_django_project/static/admin/css/changelists.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/dark_mode.css b/src/pe_reports/pe_reports_django_project/static/admin/css/dark_mode.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/dashboard.css b/src/pe_reports/pe_reports_django_project/static/admin/css/dashboard.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/fonts.css b/src/pe_reports/pe_reports_django_project/static/admin/css/fonts.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/forms.css b/src/pe_reports/pe_reports_django_project/static/admin/css/forms.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/login.css b/src/pe_reports/pe_reports_django_project/static/admin/css/login.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/nav_sidebar.css b/src/pe_reports/pe_reports_django_project/static/admin/css/nav_sidebar.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/responsive.css b/src/pe_reports/pe_reports_django_project/static/admin/css/responsive.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/responsive_rtl.css b/src/pe_reports/pe_reports_django_project/static/admin/css/responsive_rtl.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/rtl.css b/src/pe_reports/pe_reports_django_project/static/admin/css/rtl.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/LICENSE-SELECT2.md b/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/LICENSE-SELECT2.md old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.css b/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.min.css b/src/pe_reports/pe_reports_django_project/static/admin/css/vendor/select2/select2.min.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/css/widgets.css b/src/pe_reports/pe_reports_django_project/static/admin/css/widgets.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/LICENSE.txt b/src/pe_reports/pe_reports_django_project/static/admin/fonts/LICENSE.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/README.txt b/src/pe_reports/pe_reports_django_project/static/admin/fonts/README.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Bold-webfont.woff b/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Bold-webfont.woff old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Light-webfont.woff b/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Light-webfont.woff old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Regular-webfont.woff b/src/pe_reports/pe_reports_django_project/static/admin/fonts/Roboto-Regular-webfont.woff old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/LICENSE b/src/pe_reports/pe_reports_django_project/static/admin/img/LICENSE old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/README.txt b/src/pe_reports/pe_reports_django_project/static/admin/img/README.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/calendar-icons.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/calendar-icons.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_off.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_off.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_on.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/gis/move_vertex_on.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-addlink.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-addlink.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-alert.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-alert.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-calendar.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-calendar.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-changelink.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-changelink.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-clock.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-clock.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-deletelink.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-deletelink.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-no.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-no.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown-alt.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown-alt.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-unknown.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-viewlink.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-viewlink.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/icon-yes.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/icon-yes.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/inline-delete.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/inline-delete.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/search.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/search.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/selector-icons.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/selector-icons.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/sorting-icons.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/sorting-icons.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-add.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-add.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-arrowright.svg b/src/pe_reports/pe_reports_django_project/static/admin/img/tooltag-arrowright.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/SelectBox.js b/src/pe_reports/pe_reports_django_project/static/admin/js/SelectBox.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/SelectFilter2.js b/src/pe_reports/pe_reports_django_project/static/admin/js/SelectFilter2.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/actions.js b/src/pe_reports/pe_reports_django_project/static/admin/js/actions.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/admin/DateTimeShortcuts.js b/src/pe_reports/pe_reports_django_project/static/admin/js/admin/DateTimeShortcuts.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/admin/RelatedObjectLookups.js b/src/pe_reports/pe_reports_django_project/static/admin/js/admin/RelatedObjectLookups.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/autocomplete.js b/src/pe_reports/pe_reports_django_project/static/admin/js/autocomplete.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/calendar.js b/src/pe_reports/pe_reports_django_project/static/admin/js/calendar.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/cancel.js b/src/pe_reports/pe_reports_django_project/static/admin/js/cancel.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/change_form.js b/src/pe_reports/pe_reports_django_project/static/admin/js/change_form.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/collapse.js b/src/pe_reports/pe_reports_django_project/static/admin/js/collapse.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/core.js b/src/pe_reports/pe_reports_django_project/static/admin/js/core.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/filters.js b/src/pe_reports/pe_reports_django_project/static/admin/js/filters.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/inlines.js b/src/pe_reports/pe_reports_django_project/static/admin/js/inlines.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/jquery.init.js b/src/pe_reports/pe_reports_django_project/static/admin/js/jquery.init.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/nav_sidebar.js b/src/pe_reports/pe_reports_django_project/static/admin/js/nav_sidebar.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/popup_response.js b/src/pe_reports/pe_reports_django_project/static/admin/js/popup_response.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate.js b/src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate_init.js b/src/pe_reports/pe_reports_django_project/static/admin/js/prepopulate_init.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/urlify.js b/src/pe_reports/pe_reports_django_project/static/admin/js/urlify.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/LICENSE.txt b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/LICENSE.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.min.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/jquery/jquery.min.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/LICENSE.md b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/LICENSE.md old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/af.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/af.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ar.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ar.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/az.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/az.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bg.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bg.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bn.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bn.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bs.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/bs.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ca.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ca.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/cs.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/cs.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/da.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/da.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/de.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/de.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/dsb.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/dsb.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/el.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/el.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/en.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/en.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/es.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/es.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/et.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/et.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/eu.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/eu.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fa.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fa.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fi.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fi.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fr.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/fr.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/gl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/gl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/he.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/he.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hi.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hi.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hr.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hr.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hsb.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hsb.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hu.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hu.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hy.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/hy.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/id.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/id.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/is.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/is.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/it.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/it.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ja.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ja.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ka.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ka.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/km.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/km.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ko.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ko.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lt.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lt.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lv.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/lv.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/mk.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/mk.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ms.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ms.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nb.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nb.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ne.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ne.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/nl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ps.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ps.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt-BR.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt-BR.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/pt.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ro.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ro.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ru.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/ru.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sk.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sk.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sq.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sq.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr-Cyrl.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr-Cyrl.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sr.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sv.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/sv.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/th.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/th.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tk.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tk.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tr.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/tr.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/uk.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/uk.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/vi.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/vi.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-CN.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-CN.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-TW.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/i18n/zh-TW.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.min.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/select2/select2.full.min.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/LICENSE.txt b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/LICENSE.txt old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.min.js b/src/pe_reports/pe_reports_django_project/static/admin/js/vendor/xregexp/xregexp.min.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/css/custom.css b/src/pe_reports/pe_reports_django_project/static/css/custom.css old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/js/htmx.js b/src/pe_reports/pe_reports_django_project/static/js/htmx.js old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/static/power.svg b/src/pe_reports/pe_reports_django_project/static/power.svg old mode 100644 new mode 100755 diff --git a/src/pe_reports/pe_reports_django_project/templates/base.html b/src/pe_reports/pe_reports_django_project/templates/base.html old mode 100644 new mode 100755 index 479220fd..95f82d41 --- a/src/pe_reports/pe_reports_django_project/templates/base.html +++ b/src/pe_reports/pe_reports_django_project/templates/base.html @@ -1,9 +1,9 @@ {% load static %} - + - {% block title %} {% endblock %} - P&E Application + {% block title %}{% endblock %} - P&E Application @@ -48,74 +48,71 @@ {% url 'home' as url %} Home + Home +
  • {% url 'peBulkUpload' as url %} Add P&E Stakeholder + Add P&E Stakeholder +
    • -
  • {% url 'report_gen' as url %} Report Generator + Report Generator +
    • -
  • {% url 'stakeholder_lite' as url %} Stakeholder Lite + Stakeholder Lite +
    • -
  • {% url 'bulkupload' as url %} Bulk File Upload + Bulk File Upload +
    • {% if user.is_staff %}
  • {% url 'weekly_status' as url %} Weekly Status + Weekly Status +
    • {% endif %} @@ -123,58 +120,53 @@ -
    +

    Welcome to P&E Reports

    - -
    -
    - - {% if messages %} {% for message in messages %} - + {% endfor %}
    - {% endif %} {% block content %} {% endblock content %} - - {% block extra_js %} - - {% endblock %} + + {% endblock %} + diff --git a/src/pe_reports/pe_reports_django_project/templates/home.html b/src/pe_reports/pe_reports_django_project/templates/home.html old mode 100644 new mode 100755 diff --git a/src/pe_reports/sync_dmz_mdl.py b/src/pe_reports/sync_dmz_mdl.py new file mode 100644 index 00000000..54f7ee44 --- /dev/null +++ b/src/pe_reports/sync_dmz_mdl.py @@ -0,0 +1,62 @@ +import psycopg2 +from psycopg2 import sql +from data.config import staging_config + + +def create_database_and_user(db_name, new_user, new_password, admin_user, admin_password,admin_db, host='localhost', port='5432'): + # Connect to the PostgreSQL server using admin credentials + conn = psycopg2.connect(dbname=admin_db, user=admin_user, password=admin_password, host=host, port=port) + conn.autocommit = True # Enable autocommit to create databases and users + + try: + # Create a cursor object + cursor = conn.cursor() + + # Check if the new user exists + cursor.execute(sql.SQL("SELECT 1 FROM pg_roles WHERE rolname = %s"), [new_user]) + user_exists = cursor.fetchone() + + if not user_exists: + # Create the new user if it does not exist + cursor.execute(sql.SQL("CREATE USER {} WITH PASSWORD %s").format(sql.Identifier(new_user)), [new_password]) + print(f"User '{new_user}' created successfully.") + else: + print(f"User '{new_user}' already exists.") + + # Check if the database exists + cursor.execute(sql.SQL("SELECT 1 FROM pg_catalog.pg_database WHERE datname = %s"), [db_name]) + db_exists = cursor.fetchone() + + if not db_exists: + # Create the database if it does not exist + cursor.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(db_name))) + print(f"Database '{db_name}' created successfully.") + else: + print(f"Database '{db_name}' already exists.") + + # Grant all privileges on the new database to the new user + cursor.execute(sql.SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {}").format(sql.Identifier(db_name), sql.Identifier(new_user))) + print(f"Granted all privileges on '{db_name}' to user '{new_user}'.") + + except Exception as e: + print(f"An error occurred: {e}") + + finally: + # Close the cursor and connection + cursor.close() + conn.close() + +if __name__ == "__main__": + DMZ_DIC = staging_config(section="mdl_dmz") + PE_DIC = staging_config(section="cd_crossfeed") + # Database connection parameters + DB_NAME = str(DMZ_DIC.get("mdl_database")) # Name of the database to create + NEW_USER = str(DMZ_DIC.get("mdl_user")) # New PostgreSQL username + NEW_PASSWORD = str(DMZ_DIC.get("mdl_password")) # New PostgreSQL password + ADMIN_USER = str(PE_DIC.get("user")) # Admin PostgreSQL username + ADMIN_PASSWORD = str(PE_DIC.get("password")) # Admin PostgreSQL password + ADMIN_DB = str(PE_DIC.get("database")) + HOST = str(DMZ_DIC.get("mdl_host")) # Host where PostgreSQL is running + PORT = str(DMZ_DIC.get("mdl_port")) # Port on which PostgreSQL is listening + + create_database_and_user(DB_NAME, NEW_USER, NEW_PASSWORD, ADMIN_USER, ADMIN_PASSWORD, ADMIN_DB, HOST, PORT) diff --git a/src/pe_source/cybersixgill.py b/src/pe_source/cybersixgill.py index e2268bc7..c8b7f802 100644 --- a/src/pe_source/cybersixgill.py +++ b/src/pe_source/cybersixgill.py @@ -3,11 +3,14 @@ # Standard Python Libraries from datetime import date, datetime, timedelta import logging -import pandas as pd -import sys -import time + +# import sys +# import time import traceback +# Third-Party Libraries +import pandas as pd + from .data.pe_db.db_query_source import ( get_breaches, get_data_source_uid, @@ -19,18 +22,17 @@ insert_sixgill_topCVEs, ) from .data.sixgill.api import get_sixgill_organizations -from .data.sixgill.source import ( +from .data.sixgill.source import ( # cve_summary,; get_alerts_content, alerts, alias_organization, all_assets_list, creds, - cve_summary, - get_alerts_content, mentions, root_domains, top_cves, ) -from .data.helpers.redact_pii import redact_pii, redact_pii_new + +# from .data.helpers.redact_pii import redact_pii # Set todays date formatted YYYY-MM-DD and the start_date 30 days prior TODAY = date.today() @@ -48,7 +50,7 @@ LOGGER = logging.getLogger(__name__) # Suppress output from presidio-analyzer import (PII filter) -logging.getLogger('presidio-analyzer').setLevel(logging.CRITICAL+1) +logging.getLogger("presidio-analyzer").setLevel(logging.CRITICAL + 1) class Cybersixgill: @@ -116,7 +118,7 @@ def run_cybersixgill(self): # Get sixgill_org_id associated with the PE org try: sixgill_org_id = sixgill_orgs[org_id][0] - except KeyError as err: + except KeyError: LOGGER.warning(f"{org_id} is not registered in Cybersixgill") # print(err, file=sys.stderr) # failed.append("%s not in sixgill" % org_id) @@ -124,7 +126,9 @@ def run_cybersixgill(self): # Run alerts if "alerts" in method_list: - LOGGER.info(f"Fetching alert data for {org_id} ({org_idx+1} of {len(pe_orgs_final)})") + LOGGER.info( + f"Fetching alert data for {org_id} ({org_idx + 1} of {len(pe_orgs_final)})" + ) if ( self.get_alerts( org_id, @@ -138,7 +142,9 @@ def run_cybersixgill(self): failed.append("%s alerts" % org_id) # Run mentions if "mentions" in method_list: - LOGGER.info(f"Fetching mention data for {org_id} ({org_idx+1} of {len(pe_orgs_final)})") + LOGGER.info( + f"Fetching mention data for {org_id} ({org_idx + 1} of {len(pe_orgs_final)})" + ) if ( self.get_mentions( org_id, @@ -152,7 +158,9 @@ def run_cybersixgill(self): failed.append("%s mentions" % org_id) # Run credentials if "credentials" in method_list: - LOGGER.info(f"Fetching credential data for {org_id} ({org_idx+1} of {len(pe_orgs_final)})") + LOGGER.info( + f"Fetching credential data for {org_id} ({org_idx + 1} of {len(pe_orgs_final)})" + ) if ( self.get_credentials( org_id, sixgill_org_id, pe_org_uid, source_uid @@ -386,20 +394,28 @@ def get_credentials(self, org_id, sixgill_org_id, pe_org_uid, source_uid): # Catch no root assets situation if len(roots) == 0: - LOGGER.warning(f"{org_id} does not have any root domain assets in Cybersixgill") + LOGGER.warning( + f"{org_id} does not have any root domain assets in Cybersixgill" + ) return 0 # Fetch credential data if len(roots) > 100: # Catch situation where an org has >100 roots in sixgill - LOGGER.info(f"{org_id} has more than 100 root assets in cybersixgill, breaking into chunks of 100...") - root_chunks = [roots[i:i + 100] for i in range(0, len(roots), 100)] + LOGGER.info( + f"{org_id} has more than 100 root assets in cybersixgill, breaking into chunks of 100..." + ) + root_chunks = [roots[i : i + 100] for i in range(0, len(roots), 100)] creds_df = pd.DataFrame() for idx, chunk in enumerate(root_chunks): try: - LOGGER.info(f"On chunk {idx+1} of {len(root_chunks)} for {org_id} credentials") + LOGGER.info( + f"On chunk {idx+1} of {len(root_chunks)} for {org_id} credentials" + ) chunk_creds_df = creds(chunk, START_DATE_TIME, END_DATE_TIME) - LOGGER.info("Found %s credentials for this chunk", len(chunk_creds_df.index)) + LOGGER.info( + "Found %s credentials for this chunk", len(chunk_creds_df.index) + ) chunk_creds_df["organizations_uid"] = pe_org_uid chunk_creds_df["data_source_uid"] = source_uid creds_df = creds_df.append(chunk_creds_df, ignore_index=True) @@ -535,7 +551,7 @@ def get_topCVEs(self, source_uid): # except Exception: # summary = "" # top_cve_df.at[cve_index, "summary"] = summary - + except Exception as e: LOGGER.error("Failed fetching top CVEs.") LOGGER.error(e) diff --git a/src/pe_source/data/pe_db/db_query_source.py b/src/pe_source/data/pe_db/db_query_source.py index 14192d6a..275b7345 100644 --- a/src/pe_source/data/pe_db/db_query_source.py +++ b/src/pe_source/data/pe_db/db_query_source.py @@ -17,7 +17,7 @@ import requests # cisagov Libraries -from pe_reports import app +# from pe_reports import app from pe_reports.data.config import config, staging_config from pe_reports.data.db_query import task_api_call @@ -611,6 +611,36 @@ def insert_or_update_business_unit(business_unit_dict): LOGGER.error(err) +# --- Issue 699 pe-reports --- +def get_linked_xpanse_business_units(): + """ + Query API to retrieve data for all business units that link to an org. + + Return: + All linked xpanse business units + """ + # Endpoint info + endpoint_url = pe_api_url + "linked_xpanse_business_units" + headers = { + "Content-Type": "application/json", + "access_token": pe_api_key, + } + try: + result = requests.get(endpoint_url, headers=headers).json() + # Process data and return + return result + except requests.exceptions.HTTPError as errh: + LOGGER.error(errh) + except requests.exceptions.ConnectionError as errc: + LOGGER.error(errc) + except requests.exceptions.Timeout as errt: + LOGGER.error(errt) + except requests.exceptions.RequestException as err: + LOGGER.error(err) + except json.decoder.JSONDecodeError as err: + LOGGER.error(err) + + # --- Issue 682 --- def api_xpanse_alert_insert(xpanse_alert_dict): """ @@ -976,9 +1006,13 @@ def get_orgs(): # Process data and return for row in result: if row.get("date_first_reported") is not None: - row["date_first_reported"] = datetime.strptime(row.get("date_first_reported"), "%Y-%m-%d") + row["date_first_reported"] = datetime.strptime( + row.get("date_first_reported"), "%Y-%m-%d" + ) if row.get("cyhy_period_start") is not None: - row["cyhy_period_start"] = datetime.strptime(row.get("cyhy_period_start"), "%Y-%m-%d") + row["cyhy_period_start"] = datetime.strptime( + row.get("cyhy_period_start"), "%Y-%m-%d" + ) if row.get("county_fips") is not None: row["county_fips"] = Decimal(row.get("county_fips")) if row.get("state_fips") is not None: @@ -1119,10 +1153,10 @@ def org_root_domains(org_uid): result = requests.post(endpoint_url, headers=headers, data=data).json() # Process data and return result_df = pd.DataFrame.from_dict(result) - result_df.rename(columns={ - "root_domain_uid": "root_uid", - "organizations_uid": "org_uid" - }, inplace=True) + result_df.rename( + columns={"root_domain_uid": "root_uid", "organizations_uid": "org_uid"}, + inplace=True, + ) result_dict_list = result_df.to_dict("records") return result_dict_list except requests.exceptions.HTTPError as errh: @@ -1414,7 +1448,7 @@ def query_PE_subs(org_uid): # --- Issue 016 atc-framework --- -def insert_shodan_assets(dataframe, table, thread, org_name, failed): +def insert_shodan_assets(asset_data, failed): """ Query API to insert Shodan data into the shodan_assets table. @@ -1422,31 +1456,37 @@ def insert_shodan_assets(dataframe, table, thread, org_name, failed): data: Dataframe of the shodan data to be inserted into shodan_assets. """ # Endpoint info - endpoint_url = pe_api_url + "shodan_assets_inserts" + endpoint_url = pe_api_url + "shodan_assets_insert" headers = { "Content-Type": "application/json", "access_token": pe_api_key, } - data = json.dumps({"asset_data": dataframe}) + data = json.dumps({"asset_data": asset_data}) try: # Call endpoint - result = requests.post(endpoint_url, headers=headers, data=data).json() + result = requests.put(endpoint_url, headers=headers, data=data).json() # Process data and return LOGGER.info(result) except requests.exceptions.HTTPError as errh: LOGGER.error(errh) + failed.append("Failed inserting shodan assets: {}".format(errh)) except requests.exceptions.ConnectionError as errc: LOGGER.error(errc) + failed.append("Failed inserting shodan assets: {}".format(errc)) except requests.exceptions.Timeout as errt: LOGGER.error(errt) + failed.append("Failed inserting shodan assets: {}".format(errt)) except requests.exceptions.RequestException as err: LOGGER.error(err) + failed.append("Failed inserting shodan assets: {}".format(err)) except json.decoder.JSONDecodeError as err: LOGGER.error(err) + failed.append("Failed inserting shodan assets: {}".format(err)) + return failed # --- Issue 017 atc-framework --- -def insert_shodan_vulns(dataframe, table, thread, org_name, failed): +def insert_shodan_vulns(vuln_data, failed): """ Query API to insert Shodan data into the shodan_vulns table. @@ -1454,34 +1494,40 @@ def insert_shodan_vulns(dataframe, table, thread, org_name, failed): data: Dataframe of the shodan data to be inserted into shodan_vulns. """ # Endpoint info - endpoint_url = pe_api_url + "shodan_vulns_inserts" + endpoint_url = pe_api_url + "shodan_vulns_insert" headers = { "Content-Type": "application/json", "access_token": pe_api_key, } - data = json.dumps({"vuln_data": dataframe}) + data = json.dumps({"vuln_data": vuln_data}) try: # Call endpoint - result = requests.post(endpoint_url, headers=headers, data=data).json() + result = requests.put(endpoint_url, headers=headers, data=data).json() # Process data and return LOGGER.info(result) except requests.exceptions.HTTPError as errh: LOGGER.error(errh) + failed.append("Failed inserting shodan assets: {}".format(errh)) except requests.exceptions.ConnectionError as errc: LOGGER.error(errc) + failed.append("Failed inserting shodan assets: {}".format(errc)) except requests.exceptions.Timeout as errt: LOGGER.error(errt) + failed.append("Failed inserting shodan assets: {}".format(errt)) except requests.exceptions.RequestException as err: LOGGER.error(err) + failed.append("Failed inserting shodan assets: {}".format(err)) except json.decoder.JSONDecodeError as err: LOGGER.error(err) + failed.append("Failed inserting shodan assets: {}".format(err)) + return failed # v ===== ACTIVE TSQL THAT STILL NEEDS CONVERSION ===== v # Conversion in progress -def get_ips(org_uid): +def get_ips_old(org_uid): """Get IP data.""" conn = connect() sql1 = """SELECT i.ip_hash, i.ip, ct.network FROM ips i @@ -1518,6 +1564,36 @@ def get_ips(org_uid): return ips +def get_ips(org_uid): + """ + Query API to get all ips for an org to run through Shodan. + + Return: + All ips to run through Shodan + """ + # Endpoint info + endpoint_url = pe_api_url + "query_shodan_ips/" + org_uid + headers = { + "Content-Type": "application/json", + "access_token": pe_api_key, + } + try: + result = requests.get(endpoint_url, headers=headers).json() + # Process data and return + print(result) + return result + except requests.exceptions.HTTPError as errh: + LOGGER.error(errh) + except requests.exceptions.ConnectionError as errc: + LOGGER.error(errc) + except requests.exceptions.Timeout as errt: + LOGGER.error(errt) + except requests.exceptions.RequestException as err: + LOGGER.error(err) + except json.decoder.JSONDecodeError as err: + LOGGER.error(err) + + # ??? def query_orgs_rev(): """Query orgs in reverse.""" @@ -1901,12 +1977,13 @@ def insert_intelx_credentials_tsql(df): conn.rollback() cursor.close() + # WAS queries def getPotential(org_id): """Get findings for specific time period in months.""" conn = connect() cur = conn.cursor() - sql = """ SELECT COUNT(*) FROM was_findings + sql = """ SELECT COUNT(*) FROM was_findings WHERE was_org_id = '{}' AND potential IS TRUE; """ cur.execute(sql.format(org_id), conn) @@ -1927,7 +2004,7 @@ def insertWASIds(listIds): ON CONFLICT (was_org_id) DO NOTHING;""" cur = conn.cursor() for id in listIds: - if id[1] == '': + if id[1] == "": cur.execute(sqlNoUUID.format(id[0])) else: cur.execute(sql.format(id[0], id[1])) @@ -1935,26 +2012,28 @@ def insertWASIds(listIds): close(conn) print("Success adding WAS IDs to database.") -def getPreviousFindingsHistorical(org_id,monthsAgo): + +def getPreviousFindingsHistorical(org_id, monthsAgo): """Get findings for specific time period in months.""" conn = connect() cur = conn.cursor() - sql = """ SELECT * FROM was_findings + sql = """ SELECT * FROM was_findings WHERE was_org_id = '{}'; """ - cur.execute(sql.format(org_id,monthsAgo-1,monthsAgo), conn) + cur.execute(sql.format(org_id, monthsAgo - 1, monthsAgo), conn) ret = cur.fetchall() cur.close() close(conn) return ret + def insertFindingData(findingList): """Insert finding data into database.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available conn = connect() sql = """INSERT INTO was_findings (finding_uid, finding_type, webapp_id, webapp_url, webapp_name, was_org_id, name, owasp_category, severity, times_detected, cvss_v3_attack_vector, base_score, temporal_score, fstatus, last_detected, first_detected, potential, cwe_list, wasc_list) VALUES ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}') - ON CONFLICT (finding_uid) DO UPDATE + ON CONFLICT (finding_uid) DO UPDATE SET is_remidiated = CASE WHEN was_findings.fstatus != 'FIXED' AND excluded.fstatus = 'FIXED' THEN TRUE ELSE was_findings.is_remidiated @@ -1972,7 +2051,7 @@ def insertFindingData(findingList): ;""" cur = conn.cursor() for finding in findingList: - finding['cwe_list'] = '{' + ','.join(map(str,finding["cwe_list"])) + '}' + finding["cwe_list"] = "{" + ",".join(map(str, finding["cwe_list"])) + "}" try: cur.execute( sql.format( @@ -1994,7 +2073,7 @@ def insertFindingData(findingList): finding["first_detected"], finding["potential"], finding["cwe_list"], - json.dumps(finding["wasc_list"]) + json.dumps(finding["wasc_list"]), ) ) except KeyError: @@ -2004,6 +2083,7 @@ def insertFindingData(findingList): close(conn) print("Success adding finding data to database.") + def queryVulnWebAppCount(org_id): """Query the amount of webapps with vulnerabilities.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available @@ -2022,6 +2102,7 @@ def queryVulnWebAppCount(org_id): close(conn) return len(set(webIdsList)) + def queryWASOrgList(): """Query the list of WAS orgs.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available @@ -2034,7 +2115,7 @@ def queryWASOrgList(): def getPEuuid(org_id): - """Query the org uuid given a certain cyhy db name""" + """Query the org uuid given a certain cyhy db name.""" conn = connect() sql = """SELECT organizations_uid FROM organizations WHERE cyhy_db_name = '{}'""" cur = conn.cursor() @@ -2043,21 +2124,23 @@ def getPEuuid(org_id): close(conn) return ret -def getPreviousFindings(org_id,monthsAgo): + +def getPreviousFindings(org_id, monthsAgo): """Get findings for specific time period in months.""" conn = connect() cur = conn.cursor() - sql = """ SELECT * FROM was_findings + sql = """ SELECT * FROM was_findings WHERE was_org_id = '{}' AND last_detected >= date_trunc('month', now() - interval '{} month') AND last_detected < date_trunc('month', now() - interval '{} month'); """ - cur.execute(sql.format(org_id,monthsAgo,monthsAgo-1), conn) + cur.execute(sql.format(org_id, monthsAgo, monthsAgo - 1), conn) ret = cur.fetchall() cur.close() close(conn) return ret + def queryVulnCountAll(org_id): """Query the amount of webapps with vulnerabilities.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available @@ -2076,7 +2159,8 @@ def queryVulnCountAll(org_id): close(conn) return len(webIdsList) -def queryVulnCountSeverity(org_id,severity): + +def queryVulnCountSeverity(org_id, severity): """Query the amount of webapps with vulnerabilities.""" # TODO: Dont use was_ord_id to reference orgs, use customer_id once was data become available conn = connect() @@ -2090,11 +2174,12 @@ def queryVulnCountSeverity(org_id,severity): OR fstatus = 'REOPENED' ); """ - df = pd.read_sql_query(sql.format(org_id,severity), conn) + df = pd.read_sql_query(sql.format(org_id, severity), conn) webIdsList = df["webapp_id"].values.tolist() close(conn) return len(webIdsList) + def insertWASVulnData(data): """Insert WAS vulnerability data into database.""" conn = connect() @@ -2103,23 +2188,23 @@ def insertWASVulnData(data): VALUES ('{}','{}',{},{},{}, (CASE WHEN {} = 0 THEN NULL ELSE {} END), (CASE WHEN {} = 0 THEN NULL ELSE {} END),'{}',{},{},{},{},{}) """ cur.execute( sql.format( - data['was_org_id'], - data['date_scanned'], - data['vuln_cnt'], - data['vuln_webapp_cnt'], - data['web_app_cnt'], - data['high_rem_time'], - data['high_rem_time'], - data['crit_rem_time'], - data['crit_rem_time'], - data['report_period'], - data['high_vuln_cnt'], - data['crit_vuln_cnt'], - data['high_rem_cnt'], - data['crit_rem_cnt'], - data['total_potential'], - ) + data["was_org_id"], + data["date_scanned"], + data["vuln_cnt"], + data["vuln_webapp_cnt"], + data["web_app_cnt"], + data["high_rem_time"], + data["high_rem_time"], + data["crit_rem_time"], + data["crit_rem_time"], + data["report_period"], + data["high_vuln_cnt"], + data["crit_vuln_cnt"], + data["high_rem_cnt"], + data["crit_rem_cnt"], + data["total_potential"], ) + ) conn.commit() close(conn) print("Success adding finding data to database.") @@ -2201,7 +2286,7 @@ def getSubdomain_tsql(domain): if conn is not None: close(conn) - + # --- 703 pe-reports OLD TSQL --- def org_root_domains_tsql(conn, org_uid): """Get root domains from database given the org_uid.""" @@ -2262,12 +2347,12 @@ def getDataSource(conn, source): # --- 709 pe-reports/008 atc-framework OLD TSQL --- -# This TSQL is from the execute_hibp_breach_values() function in +# This TSQL is from the execute_hibp_breach_values() function in # hibp_latest.py and hibp_latest_rev.py # --- 710 pe-reports/009 atc-framework OLD TSQL --- -# This TSQL is from the execute_hibp_emails_values() function in +# This TSQL is from the execute_hibp_emails_values() function in # hibp_latest.py and hibp_latest_rev.py @@ -2277,12 +2362,12 @@ def getDataSource(conn, source): # --- 011 atc-framework OLD TSQL --- -# This TSQL is from the query_orgs() function in +# This TSQL is from the query_orgs() function in # adhoc/data/run.py # --- 012 atc-framework OLD TSQL --- -# This TSQL is from the query_PE_subs() function in +# This TSQL is from the query_PE_subs() function in # hibp_latest.py and hibp_latest_rev.py @@ -2291,7 +2376,7 @@ def getDataSource(conn, source): def insert_shodan_data(dataframe, table, thread, org_name, failed): """Insert Shodan data into database.""" # get rid of \x00 characters - dataframe = dataframe.replace({'\x00': ''}, regex=True) + dataframe = dataframe.replace({"\x00": ""}, regex=True) conn = connect() tpls = [tuple(x) for x in dataframe.to_numpy()] cols = ",".join(list(dataframe.columns)) @@ -2320,4 +2405,4 @@ def insert_shodan_data(dataframe, table, thread, org_name, failed): failed.append("{} failed inserting into {}".format(org_name, table)) conn.rollback() cursor.close() - return failed \ No newline at end of file + return failed diff --git a/src/pe_source/data/shodan_db/shodan_search.py b/src/pe_source/data/shodan_db/shodan_search.py index 286f9446..f465852a 100644 --- a/src/pe_source/data/shodan_db/shodan_search.py +++ b/src/pe_source/data/shodan_db/shodan_search.py @@ -6,7 +6,7 @@ import time # Third-Party Libraries -import pandas as pd +# import pandas as pd import requests import shodan @@ -14,7 +14,8 @@ from pe_source.data.pe_db.db_query_source import ( # get_ips_dhs,; get_ips_hhs,; get_ips_nasa, get_data_source_uid, get_ips, - insert_shodan_data, + insert_shodan_assets, + insert_shodan_vulns, ) LOGGER = logging.getLogger(__name__) @@ -103,6 +104,7 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): ) # Loop through chunks and query Shodan + source_uid = get_data_source_uid("Shodan") for i, ip_chunk in enumerate(ip_chunks): count = i + 1 try_count = 1 @@ -167,7 +169,8 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "is_verified": False, "cpe": d.get("cpe", None), "banner": d.get("data", None), - "version": d.get("version", None) + "version": d.get("version", None), + "data_source_uid": source_uid, } ) elif d["_shodan"]["module"] in risky_ports: @@ -211,7 +214,8 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "is_verified": False, "cpe": d.get("cpe", None), "banner": d.get("data", None), - "version": d.get("version", None) + "version": d.get("version", None), + "data_source_uid": source_uid, } ) @@ -231,7 +235,8 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): "tags": r["tags"], "timestamp": d["timestamp"], "country_code": location["country_code"], - "location": str(location) + "location": str(location), + "data_source_uid": source_uid, } ) @@ -269,22 +274,12 @@ def search_shodan(thread_name, ips, api, start, end, org_uid, org_name, failed): LOGGER.info("{} {}/{} complete - {}".format(thread_name, count, tot, org_name)) - df = pd.DataFrame(data) - risk_df = pd.DataFrame(risk_data) - vuln_df = pd.DataFrame(vuln_data) - all_vuln_df = vuln_df.append(risk_df, ignore_index=True) + all_vulns = vuln_data + risk_data # Grab the data source uid and add to each dataframe - source_uid = get_data_source_uid("Shodan") - df["data_source_uid"] = source_uid - risk_df["data_source_uid"] = source_uid - vuln_df["data_source_uid"] = source_uid - all_vuln_df["data_source_uid"] = source_uid # Insert data into the PE database - failed = insert_shodan_data(df, "shodan_assets", thread_name, org_name, failed) - failed = insert_shodan_data( - all_vuln_df, "shodan_vulns", thread_name, org_name, failed - ) + failed = insert_shodan_assets(data, failed) + failed = insert_shodan_vulns(all_vulns, failed) return failed diff --git a/src/pe_source/intelx_identity.py b/src/pe_source/intelx_identity.py index 5462f29b..19d0324a 100644 --- a/src/pe_source/intelx_identity.py +++ b/src/pe_source/intelx_identity.py @@ -11,12 +11,10 @@ import requests from .data.pe_db.config import get_params -from .data.pe_db.db_query_source import ( - connect, +from .data.pe_db.db_query_source import ( # get_intelx_breaches,; connect, get_data_source_uid, - get_intelx_breaches, get_orgs, - get_root_domains, + get_root_domains_api, insert_intelx_breaches, insert_intelx_credentials, ) @@ -77,21 +75,25 @@ def run_intelx(self): # Verify the org is in the list of orgs to scan if cyhy_org_id in orgs_list or orgs_list == "all" or orgs_list == "DEMO": - LOGGER.info(f"Running IntelX on {cyhy_org_id} ({org_idx+1} of {len(pe_orgs_final)})") + LOGGER.info( + f"Running IntelX on {cyhy_org_id} ({org_idx + 1} of {len(pe_orgs_final)})" + ) if self.get_credentials(cyhy_org_id, pe_org_uid) == 1: LOGGER.error("Failed to get credentials for %s", cyhy_org_id) failed += 1 else: - success +=1 - - LOGGER.info(f"IntelX scan ran successfully for {success}/{len(pe_orgs_final)} organizations") + success += 1 + LOGGER.info( + f"IntelX scan ran successfully for {success}/{len(pe_orgs_final)} organizations" + ) def get_credentials(self, cyhy_org_id, pe_org_uid): """Get credentials for a provided org.""" try: - conn = connect() - roots_df = get_root_domains(conn, pe_org_uid) + # conn = connect() + roots_df = get_root_domains_api(pe_org_uid) + except Exception as e: LOGGER.error("Failed fetching root domains for %s", cyhy_org_id) LOGGER.error(e) @@ -100,6 +102,7 @@ def get_credentials(self, cyhy_org_id, pe_org_uid): leaks_json = self.find_credential_leaks( roots_df["root_domain"].values.tolist(), START_DATE, END_DATE ) + print(leaks_json) if len(leaks_json) < 1: LOGGER.info(f"No credentials found for {cyhy_org_id}") return 0 @@ -112,11 +115,11 @@ def get_credentials(self, cyhy_org_id, pe_org_uid): LOGGER.error(e) return 1 - breach_dict = get_intelx_breaches(SOURCE_UID) - breach_dict = dict(breach_dict) - for cred_index, cred_row in creds_df.iterrows(): - breach_uid = breach_dict[cred_row["breach_name"]] - creds_df.at[cred_index, "credential_breaches_uid"] = breach_uid + # breach_dict = get_intelx_breaches(SOURCE_UID) + # breach_dict = dict(breach_dict) + # for cred_index, cred_row in creds_df.iterrows(): + # breach_uid = breach_dict[cred_row["breach_name"]] + # creds_df.at[cred_index, "credential_breaches_uid"] = breach_uid try: insert_intelx_credentials(creds_df) except Exception as e: @@ -314,4 +317,4 @@ def process_leaks_results(self, leaks_json, org_uid): ] ] - return creds_df, breaches_df \ No newline at end of file + return creds_df, breaches_df diff --git a/src/pe_source/pe_scripts.py b/src/pe_source/pe_scripts.py index eb5a98c8..24e6fd75 100644 --- a/src/pe_source/pe_scripts.py +++ b/src/pe_source/pe_scripts.py @@ -5,7 +5,7 @@ Arguments: DATA_SOURCE Source to collect data from. Valid values are "cybersixgill", - "dnstwist", "hibp", "intelx", "pshtt", and "shodan". + "dnstwist", "xpanse" "hibp", "intelx", "pshtt", and "shodan". Options: -h --help Show this message. @@ -28,11 +28,9 @@ # Standard Python Libraries import logging import sys -import time from typing import Any, Dict # Third-Party Libraries -from datetime import timedelta import docopt from schema import And, Schema, SchemaError, Use @@ -46,6 +44,7 @@ from .intelx_identity import IntelX from .pshtt_wrapper import launch_pe_pshtt from .shodan_wrapper import Get_shodan +from .xpanse_alert_pull import run_xpanse_scans LOGGER = logging.getLogger(__name__) @@ -53,65 +52,34 @@ def run_pe_script(source, orgs_list, cybersix_methods, soc_med_included): """Collect data from the source specified.""" # If not "all", separate orgs string into a list of orgs - if orgs_list != "all" and orgs_list != "DEMO": + if orgs_list != "all" and orgs_list != "DEMO" and source != "xpanse": orgs_list = orgs_list.split(",") # If not "all", separate Cybersixgill methods string into a list - sixgill_scan_name = cybersix_methods.title() if cybersix_methods == "all": cybersix_methods = ["alerts", "mentions", "credentials", "topCVEs"] else: cybersix_methods = cybersix_methods.split(",") - # LOGGER.info("Running %s on these orgs: %s", source, orgs_list) + LOGGER.info("Running %s on these orgs: %s", source, orgs_list) if source == "cybersixgill": - if sixgill_scan_name == "Topcves": - sixgill_scan_name = "Top CVEs" - LOGGER.info(f"--- Cybersixgill {sixgill_scan_name} Scan Starting ---") - LOGGER.info(f"Running Cybersixgill {sixgill_scan_name} on these orgs: {orgs_list}") - sixgill_start_time = time.time() cybersix = Cybersixgill(orgs_list, cybersix_methods, soc_med_included) cybersix.run_cybersixgill() - sixgill_end_time = time.time() - LOGGER.info(f"Execution time for Cybersixgill {sixgill_scan_name} scan: {str(timedelta(seconds=(sixgill_end_time - sixgill_start_time)))} (H:M:S)") - LOGGER.info(f"--- Cybersixgill {sixgill_scan_name} Scan Complete ---") elif source == "shodan": - LOGGER.info("--- Shodan Scan Starting ---") - LOGGER.info(f"Running Shodan on these orgs: {orgs_list}") - shodan_start_time = time.time() shodan = Get_shodan(orgs_list) shodan.run_shodan() - shodan_end_time = time.time() - LOGGER.info(f"Execution time for Shodan scan: {str(timedelta(seconds=(shodan_end_time - shodan_start_time)))} (H:M:S)") - LOGGER.info("--- Shodan Scan Complete ---") elif source == "dnsmonitor": - LOGGER.info("--- DNSMonitor Scan Starting ---") - LOGGER.info(f"Running DNSMonitor on these orgs: {orgs_list}") - dnsmonitor_start_time = time.time() dnsMonitor = DNSMonitor(orgs_list) dnsMonitor.run_dnsMonitor() - dnsmonitor_end_time = time.time() - LOGGER.info(f"Execution time for DNSMonitor scan: {str(timedelta(seconds=(dnsmonitor_end_time - dnsmonitor_start_time)))} (H:M:S)") - LOGGER.info("--- DNSMonitor Scan Complete ---") elif source == "dnstwist": - LOGGER.info("--- DNSTwist Scan Starting ---") - LOGGER.info(f"Running DNSTwist on these orgs: {orgs_list}") - dnstwist_start_time = time.time() run_dnstwist(orgs_list) - dnstwist_end_time = time.time() - LOGGER.info(f"Execution time for DNSTwist scan: {str(timedelta(seconds=(dnstwist_end_time - dnstwist_start_time)))} (H:M:S)") - LOGGER.info("--- DNSTwist Scan Complete ---") elif source == "intelx": - LOGGER.info("--- IntelX Scan Starting ---") - LOGGER.info(f"Running IntelX on these orgs: {orgs_list}") - intelx_start_time = time.time() intelx = IntelX(orgs_list) intelx.run_intelx() - intelx_end_time = time.time() - LOGGER.info(f"Execution time for IntelX scan: {str(timedelta(seconds=(intelx_end_time - intelx_start_time)))} (H:M:S)") - LOGGER.info("--- IntelX Scan Complete ---") elif source == "pshtt": launch_pe_pshtt() + elif source == "xpanse": + run_xpanse_scans("", orgs_list) else: logging.error( "Not a valid source name. Correct values are cybersixgill or shodan." diff --git a/src/pe_source/shodan_wrapper.py b/src/pe_source/shodan_wrapper.py index d2e441e6..c34ed677 100644 --- a/src/pe_source/shodan_wrapper.py +++ b/src/pe_source/shodan_wrapper.py @@ -1,21 +1,24 @@ """Collect Shodan data.""" # Standard Python Libraries +# from datetime import timedelta import logging import threading -import time # Third-Party Libraries -from datetime import timedelta import numpy from .data.pe_db.config import shodan_api_init from .data.pe_db.db_query_source import get_orgs from .data.shodan_db.shodan_search import run_shodan_thread +# import time + + # Logging LOGGER = logging.getLogger(__name__) + class Get_shodan: """Fetch Shodan data.""" @@ -29,7 +32,6 @@ def run_shodan(self): # Get orgs from PE database pe_orgs = get_orgs() - # Filter orgs if specified pe_orgs_final = [] if orgs_list == "all": @@ -61,7 +63,7 @@ def run_shodan(self): i = 0 thread_list = [] while i < len(chunked_orgs_list): - thread_name = f"Thread {i+1}:" + thread_name = f"Thread {i + 1}:" # Start thread t = threading.Thread( target=run_shodan_thread, @@ -73,4 +75,4 @@ def run_shodan(self): # Wait until all threads finish to continue for thread in thread_list: - thread.join() \ No newline at end of file + thread.join() diff --git a/src/pe_source/xpanse_alert_pull.py b/src/pe_source/xpanse_alert_pull.py new file mode 100644 index 00000000..c08f20fd --- /dev/null +++ b/src/pe_source/xpanse_alert_pull.py @@ -0,0 +1,489 @@ +"""Scan to track Xpanse alerts and incidents in the P&E database. + +Usage: + xpanse_alert_pull.py [--orgs=ORG_LIST] [--last_modified=MOD_TIME] [--log-level=LEVEL] + +Options: + -h --help Show this message. + -o --orgs=ORG_LIST A semicolon-separated list of Xpanse business_units. + If not specified, data will be gathered for all business_units. + Orgs in the list must match the names in Xpanse. E.g. Culberson County, Texas; DHS - Citizenship and Immigration Services (CIS) - CISA + [default: all] + -m --last_modified=MOD_TIME An integer in timestamp epoch milliseconds. + Scan will pull all alerts and assets updated since the provided time. + If not specified, data will be gathered for all assets and alerts. E.g. 1696996800000 + [default: all_time] + -l --log-level=LEVEL If specified, then the log level will be set to + the specified value. Valid values are "debug", "info", + "warning", "error", and "critical". [default: info] +""" + +# Standard Python Libraries +import datetime +import json +import logging +import time + +# Third-Party Libraries +# import docopt +import pytz +import requests + +# cisagov Libraries +# import pe_reports +from pe_reports.data.config import staging_config + +# from _version import __version__ +from .data.pe_db.db_query_source import ( # api_pull_xpanse_vulns, + api_xpanse_alert_insert, + get_linked_xpanse_business_units, +) + +# import sys +# from typing import Any, Dict + + +# from schema import And, Or, Schema, SchemaError, Use + + +API_DIC = staging_config(section="xpanse") +xpanse_url = "https://api-cisa-xpanse.crtx.gv.paloaltonetworks.com/public_api/" +api_key = API_DIC.get("api_key") +auth_id = API_DIC.get("auth_id") + +LOGGER = logging.getLogger(__name__) + + +# def pull_asset_data(xpanse_asset_id_list=[]): +# """Pull asset data from the Xpanse API.""" +# assets = [] + +# url = xpanse_url + "v1/assets/get_asset_internet_exposure" +# request_data = {"asm_id_list": xpanse_asset_id_list} + +# payload = json.dumps({"request_data": request_data}) + +# headers = { +# "x-xdr-auth-id": auth_id, +# "Authorization": api_key, +# "Content-Type": "application/json", +# } + +# response = requests.request("POST", url, headers=headers, data=payload) + +# resp_dict = response.json() + +# for asset in resp_dict["reply"]["details"]: +# asset_dict = format_asset(asset) +# assets.append(asset_dict) + +# return assets +# # save_asset(asset_dict) + + +def pull_alerts_data(linked_org_list, business_units_list=[]): + """Pull alerts data from the Xpanse API.""" + url = xpanse_url + "v2/alerts/get_alerts_multi_events" + + if len(business_units_list) == 0: + business_units_list = list(map(lambda d: d["entity_name"], linked_org_list)) + + for org in business_units_list: + request_data = {"use_page_token": True, "search_from": 0, "search_to": 5000} + filters = [] + LOGGER.info("Running Xpanse alert pull on %s", org) + filters.append( + {"field": "business_units_list", "operator": "in", "value": [org]} + ) + + # TODO maybe change this to be creation time + # if last_modified != "all_time": + # filters.append({ + # "field": "" + # }) + + if len(filters) > 0: + request_data["filters"] = filters + + payload = json.dumps({"request_data": request_data}) + + headers = { + "x-xdr-auth-id": auth_id, + "Authorization": api_key, + "Content-Type": "application/json", + } + try: + response = requests.request("POST", url, headers=headers, data=payload) + resp_dict = response.json() + print(resp_dict) + + page_token = resp_dict["reply"]["next_page_token"] + LOGGER.info( + "The current org has %s alerts", resp_dict["reply"]["total_count"] + ) + + formatted_alerts = format_alerts(resp_dict["reply"]["alerts"]) + + for alert in formatted_alerts: + api_xpanse_alert_insert(alert) + + while page_token is not None: + request_data = {"next_page_token": page_token} + + payload = json.dumps({"request_data": request_data}) + + response = requests.request("POST", url, headers=headers, data=payload) + resp_dict = response.json() + + page_token = resp_dict["reply"]["next_page_token"] + + formatted_alerts = format_alerts(resp_dict["reply"]["alerts"]) + for alert in formatted_alerts: + api_xpanse_alert_insert(alert) + + LOGGER.info("Done Xpanse alert pull on %s", org) + except Exception as e: + LOGGER.error("Error querying assets for %s: %s.", org, e) + + +# def format_asset(asset): +# """Format Xpanse asset to match db tables.""" +# asset_dict = { +# "asm_id": asset.get("asm_ids", None), +# "asset_name": asset.get("name", None), +# "asset_type": asset.get("type", None), +# "last_observed": asset.get("last_observed", None), +# "first_observed": asset.get("first_observed", None), +# "externally_detected_providers": asset.get( +# "externally_detected_providers", None +# ), +# "created": asset.get("created", None), +# "ips": asset.get("ips", None), +# "active_external_services_types": asset.get( +# "active_external_services_types", None +# ), +# "domain": asset.get("domain", None), +# "certificate_issuer": asset.get("certificate_issuer", None), +# "certificate_algorithm": asset.get("certificate_algorithm", None), +# "certificate_classifications": asset.get("certificate_classifications", None), +# "resolves": asset.get("resolves", None), +# "top_level_asset_mapper_domain": asset["details"].get( +# "topLevelAssetMapperDomain", None +# ), +# "domain_asset_type": asset["details"].get("domainAssetType", None), +# "is_paid_level_domain": asset["details"].get("isPaidLevelDomain", None), +# "domain_details": asset["details"].get("domainDetails", None), +# "dns_zone": asset.get("dnsZone", None), +# "latest_sampled_ip": asset.get("latestSampledIp", None), +# "recent_ips": asset.get("recentIps", None), +# "external_services": asset.get("external_services", None), +# "externally_inferred_vulnerability_score": asset.get( +# "externally_inferred_vulnerability_score", None +# ), +# "externally_inferred_cves": asset.get("externally_inferred_cves", None), +# "explainers": asset.get("explainers", None), +# "tags": asset.get("tags", None), +# } + +# return asset_dict + + +def format_alerts(alerts): + """Format Xpanse alerts to match db tables.""" + alert_services_dict = {} + service_ids = [] + for alert in alerts: + try: + service_ids += alert.get("service_ids", []) + alert_services_dict[alert["alert_id"]] = alert.get("service_ids", []) + except Exception: + continue + + services = [] + max_n = 5000 + if service_ids is not None: + service_id_chunks = [ + service_ids[i : i + max_n] for i in range(0, len(service_ids), max_n) + ] + + for service_chunk in service_id_chunks: + max_retries = 3 + retry_delay = 5 + for retry_count in range(max_retries): + try: + service_response = pull_service_data(service_chunk) + if service_response is not None: + break + except Exception as e: + # Log the error message + LOGGER.error(f"Error querying services: {e}") + + # If it's not the last retry, wait for the retry_delay before retrying + if retry_count < max_retries - 1: + LOGGER.info("Retrying...") + time.sleep(retry_delay) + else: + # If it's the last retry, set it to None and it will skip to the next chunk + service_response = None + + if service_response is None: + continue + for service_obj in service_response: + cves = [] + if service_obj["details"].get("inferredCvesObserved", None) is not None: + for cve in service_obj["details"].get("inferredCvesObserved", None): + cves.append( + ( + { + "cve_id": cve["inferredCve"]["cveId"], + "cvss_score_v2": cve["inferredCve"].get( + "cvssScoreV2", None + ), + "cve_severity_v2": cve["inferredCve"].get( + "cveSeverityV2", None + ), + "cvss_score_v3": cve["inferredCve"].get( + "cvssScoreV3", None + ), + "cve_severity_v3": cve["inferredCve"].get( + "cveSeverityV3", None + ), + }, + { + "inferred_cve_match_type": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("inferredCveMatchType", None), + "product": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("product", None), + "confidence": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("confidence", None), + "vendor": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("vendor", None), + "version_number": cve["inferredCve"][ + "inferredCveMatchMetadata" + ].get("version", None), + "activity_status": cve.get("activityStatus", None), + "first_observed": cve.get("firstObserved", None), + "last_observed": cve.get("lastObserved", None), + }, + ) + ) + services.append( + { + "service_id": service_obj.get("service_id", None), + "service_name": service_obj.get("service_name", None), + "service_type": service_obj.get("service_type", None), + "ip_address": service_obj.get( + "ip_address", None + ), # list of ip strings + "domain": service_obj.get("domain", None), # list of ? + "externally_detected_providers": service_obj.get( + "externally_detected_providers", None + ), + "is_active": service_obj.get("is_active", None), + "first_observed": service_obj.get("first_observed", None), + "last_observed": service_obj.get("last_observed", None), + "port": service_obj.get("port", None), + "protocol": service_obj.get("protocol", None), + "active_classifications": service_obj.get( + "active_classifications", None + ), # list of strings + "inactive_classifications": service_obj.get( + "inactive_classifications", None + ), + "discovery_type": service_obj.get("discovery_type", None), + "externally_inferred_vulnerability_score": service_obj.get( + "externally_inferred_vulnerability_score", None + ), + "externally_inferred_cves": service_obj.get( + "externally_inferred_cves", None + ), + "service_key": service_obj["details"].get("serviceKey", None), + "service_key_type": service_obj["details"].get( + "serviceKeyType", None + ), + # providerDetails + # certificates + # domains + # ips + # classifications + # tlsVersions + "cves": cves + # enrichedObservationSource + # ip_ranges + } + ) + + alert_list = [] + for alert in alerts: + tags = (alert.get("tags", None),) + business_units_list = [] + try: + for tag in tags[0]: + if tag.startswith("BU:"): + business_units_list.append(tag[3:].strip()) + # print(business_units_list) + except Exception: + business_units_list = [] + + assets = [] + # #Uncomment to track assets + # asset_ids = alert["asset_ids"] + # if asset_ids is not None: + # asset_id_chunks = [ + # asset_ids[i : i + max_n] for i in range(0, len(asset_ids), max_n) + # ] + + # for asset_chunk in asset_id_chunks: + # asset_response = pull_asset_data(asset_chunk) + # assets += asset_response + current_services = [] + try: + for service in alert.get("service_ids", []): + service_identified = next( + (d for d in services if d.get("service_id") == service), None + ) + if service_identified: + current_services.append(service_identified) + except Exception: + pass + + alert_dict = { + "time_pulled_from_xpanse": datetime.datetime.utcnow().replace( + tzinfo=pytz.utc + ), + "alert_id": alert.get("alert_id", None), + "detection_timestamp": alert.get("detection_timestamp", None), + "alert_name": alert.get("name", None), + # endpoint_id ???, + "description": alert.get("description", None), + # "endpoint_id": alert.get('endpoint_id', None), + # "host_ip": alert.get('host_ip', None), + "host_name": alert.get("host_name", None), + "alert_action": alert.get("action", None), + # user_name ??? null, + # mac_addresses ??? null, + # source ??? null, + "action_pretty": alert.get("action_pretty", None), + # category ??? null, + # project ??? null, + # cloud_provider ??? null, + # resource_sub_type ??? null, + # resource_type ??? null, + "action_country": alert.get("action_country", None), # list type + # event_type ??? null, + # is_whitelisted ??? null, + # image_name ??? null, + # action_local_ip ??? null, + # action_local_port ??? null, + # action_external_hostname ??? null, + # action_remote_ip ??? null, + "action_remote_port": alert.get("action_remote_port", None), # list type + # "matching_service_rule_id ??? null, + "starred": alert.get("starred", None), + "external_id": alert.get("external_id", None), + "related_external_id": None, + "alert_occurrence": None, + "severity": alert.get("severity", None), + "matching_status": alert.get("matching_status", None), + # end_match_attempt_ts ??? null, + "local_insert_ts": alert.get("local_insert_ts", None), + "last_modified_ts": alert.get("last_modified_ts") + if alert.get("last_modified_ts") is not None + else alert.get("local_insert_ts", None), + "case_id": alert.get("case_id", None), + # deduplicate_tokens ??? null, + # filter_rule_id ??? null, + # event_id ??? null, + "event_timestamp": alert.get("event_timestamp", None), # list type + # action_local_ip_v6 ??? null, + # action_remote_ip_v6 ??? null, + "alert_type": alert.get("alert_type", None), + "resolution_status": alert.get("resolution_status", None), + "resolution_comment": alert.get("resolution_comment", None), + # dynamic_fields ??? null, + "tags": alert.get("tags", None), + # malicious_urls ??? null, + "last_observed": alert.get("last_observed", None), + "country_codes": alert.get("country_codes", None), # list type + "cloud_providers": alert.get("cloud_providers", None), # list type + "ipv4_addresses": alert.get("ipv4_addresses", None), # list type + # ipv6_addresses ??? null, + "domain_names": alert.get("domain_names", None), # list type + "service_ids": alert.get("service_ids", None), # already addressed above + # "website_ids": alert.get('website_ids', None), + "asset_ids": alert.get("asset_ids", None), # list type + "certificate": alert.get("certificate", None), + # { + # issuerName": "IOS-Self-Signed-Certificate-782645061", + # subjectName": "IOS-Self-Signed-Certificate-782645061", + # validNotBefore": 1398850008000, + # validNotAfter": 1577836800000, + # serialNumber": "1" + # }, + "port_protocol": alert.get("port_protocol", None), + # business_unit_hierarchies + # "business_unit_hierarchies": alert.get('business_unit_hierarchies', None), #list of BUs + # attack_surface_rule_name ??? null, + # remediation_guidance ??? null, + "attack_surface_rule_name": alert.get("attack_surface_rule_name", None), + "remediation_guidance": alert.get("remediation_guidance", None), + "asset_identifiers": alert.get( + "asset_identifiers", None + ), # messy list of objects + "business_units": business_units_list, + "services": current_services, + "assets": assets, + } + + if alert_dict["external_id"] is not None: + alert_dict["related_external_id"] = "-".join( + alert_dict["external_id"].split("-")[:-1] + ) + alert_dict["alert_occurrence"] = ( + int(alert_dict["external_id"].split("-")[-1]) / 2 + ) + else: + alert_dict["related_external_id"] = None + alert_dict["alert_occurrence"] = None + + alert_list.append(alert_dict) + return alert_list + + +def pull_service_data(service_id_list): + """Pull service info from the Xpanse API using a service_id.""" + url = xpanse_url + "v1/assets/get_external_service" + request_data = {"service_id_list": service_id_list} + + payload = json.dumps({"request_data": request_data}) + + headers = { + "x-xdr-auth-id": auth_id, + "Authorization": api_key, + "Content-Type": "application/json", + } + + response = requests.request("POST", url, headers=headers, data=payload) + + resp_dict = response.json() + + return resp_dict.get("reply", {}).get("details", None) + + +def run_xpanse_scans(last_modified, orgs_list): + """Run Xpanse scans.""" + if orgs_list != "all": + orgs_list = orgs_list.split(";") + else: + orgs_list = [] + + linked_org_list = get_linked_xpanse_business_units() + + pull_alerts_data(linked_org_list, orgs_list) + # api_pull_xpanse_vulns(orgs_list[0], datetime.datetime(2023, 10, 10, 1, 00)) + + return 1 diff --git a/src/pe_source/xpanse_org_sync.py b/src/pe_source/xpanse_org_sync.py new file mode 100644 index 00000000..ca091f5a --- /dev/null +++ b/src/pe_source/xpanse_org_sync.py @@ -0,0 +1,147 @@ +"""Script to save Xpanse Business Units and link to cyhy orgs. + +Usage: + xpanse_org_sync.py XPANSE_ORG_CSV_PATH [--log-level=LEVEL] + +Options: + -h --help Show this message. + XPANSE_ORG_CSV_PATH The path to the XPANSE Business_unit CSV. + -l --log-level=LEVEL If specified, then the log level will be set to + the specified value. Valid values are "debug", "info", + "warning", "error", and "critical". [default: info] +""" +# Standard Python Libraries +import csv +import datetime +import json +import logging +import sys +from typing import Any, Dict +import re + +# Third-Party Libraries +from _version import __version__ +from data.pe_db.db_query_source import ( # api_pull_xpanse_vulns, + insert_or_update_business_unit, +) +import docopt +import pytz +import requests +from schema import And, Or, Schema, SchemaError, Use + +# cisagov Libraries +import pe_reports +from pe_reports.data.config import staging_config + +LOGGER = logging.getLogger(__name__) + +def extract_last_substring_in_square_brackets(input_string): + # Define the regular expression pattern + pattern = r'\[([^\]]+)\]' # Matches [ followed by any characters that are not ], followed by ] + + # Find all matches of the pattern in the input_string + matches = re.findall(pattern, input_string) + # Return the last match or None if no matches are found + return matches[-1] if matches else None + +def sync_orgs(orgs_csv): + """Sync orgs to the database.""" + try: + print(orgs_csv) + orgs_reader = csv.DictReader(orgs_csv) + except FileNotFoundError: + LOGGER.error("No file found at provided filepath.") + except Exception as e: + LOGGER.error("Unknown error reading csv: %s", e) + + # map_dict = {} + # try: + # file_path = 'xpanse_org_map.csv' + # with open(file_path, 'r', encoding='utf-8-sig') as csvfile: + # map_reader = csv.DictReader(csvfile) + + # for d in map_reader: + # print(d) + # # Add key-value pair to the result dictionary + # print(d['Xpanse Business Unit Name']) + # print(d['CISA Short Code']) + # map_dict[d['Xpanse Business Unit Name']] = d['CISA Short Code'] # Print each row as a dictionary + # except FileNotFoundError: + # print(f"Error: File '{file_path}' not found.") + # except Exception as e: + # print(f"Error reading '{file_path}': {e}") + # # Initialize an empty dictionary to store the result + # print(map_dict) + + for org in orgs_reader: + try: + # if map_dict.get(org["Entity Name"].strip(), None) is not None: + # print(map_dict.get(org["Entity Name"].strip())) + cyhy_db_name = extract_last_substring_in_square_brackets(org["Entity Name"].strip()) + if cyhy_db_name: + print(cyhy_db_name) + business_unit_dict = { + "entity_name": org["Entity Name"].strip(), + "state": org["State"].strip(), + "county": org["County"].strip(), + "city": org["City"].strip(), + "sector": org["Sector"].strip(), + "entity_type": org["Entity Type"].strip(), + "region": org["Region"].strip(), + "rating": int(org["Rating"].strip()), + "cyhy_db_name": cyhy_db_name + } + + response = insert_or_update_business_unit(business_unit_dict) + except Exception as e: + LOGGER.error('Failure saving %s', org["Entity Name"]) + LOGGER.error("Unknown error saving: %s", e) + continue + + +def main(): + """Launch Xpanse scans.""" + args: Dict[str, str] = docopt.docopt(__doc__, version=__version__) + + + schema: Schema = Schema( + { + "--log-level": And( + str, + Use(str.lower), + lambda n: n in ("debug", "info", "warning", "error", "critical"), + error="Possible values for --log-level are " + + "debug, info, warning, error, and critical.", + ), + "XPANSE_ORG_CSV_PATH": Or( + None, + Use(open, error="XPANSE_ORG_CSV_PATH should point to a readable CSV"), + ) + } + ) + + + try: + + validated_args: Dict[str, Any] = schema.validate(args) + + except SchemaError as err: + # Exit because one or more of the arguments were invalid + print(err, file=sys.stderr) + sys.exit(1) + + + log_level: str = validated_args["--log-level"] + + logging.basicConfig( + filename=pe_reports.CENTRAL_LOGGING_FILE, + filemode="a", + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%m/%d/%Y %I:%M:%S", + level=log_level.upper(), + ) + + sync_orgs(validated_args["XPANSE_ORG_CSV_PATH"]) + +if __name__ == "__main__": + main() \ No newline at end of file