Merge branch 'master' into up_master

.gitignore

@@ -58,7 +58,7 @@ cover/
 *.pot
 
 # Django stuff:
-*.log
+*.log*
 local_settings.py
 db.sqlite3
 db.sqlite3-journal

CveXplore/VERSION

@@ -1 +1 @@
-0.1.8.dev1
+0.1.8.dev2

CveXplore/main.py

@@ -13,7 +13,7 @@
 from CveXplore.common.db_mapping import database_mapping
 from CveXplore.database.connection.mongo_db import MongoDBConnection
 from CveXplore.errors import DatabaseIllegalCollection
-from CveXplore.lib.main_updater import MainUpdater
+from CveXplore.update.main_updater import MainUpdater
 
 try:
     from version import VERSION

CveXplore/update/Config.py

@@ -0,0 +1,123 @@
+import datetime
+import os
+import re
+import urllib.parse
+
+import pymongo
+import redis
+
+runPath = os.path.dirname(os.path.realpath(__file__))
+
+
+class Configuration(object):
+    CVE_START_YEAR = os.getenv("CVE_START_YEAR", 2002)
+
+    SOURCES = os.getenv(
+        "SOURCES",
+        {
+            "cve": "https://nvd.nist.gov/feeds/json/cve/1.1/",
+            "cpe": "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.json.zip",
+            "cwe": "https://cwe.mitre.org/data/xml/cwec_v4.4.xml.zip",
+            "capec": "https://capec.mitre.org/data/xml/capec_v3.4.xml",
+            "via4": "https://www.cve-search.org/feeds/via4.json",
+        },
+    )
+
+    HTTP_PROXY = os.getenv("HTTP_PROXY", "")
+
+    LOGGING_MAX_FILE_SIZE = os.getenv("LOGGING_MAX_FILE_SIZE", "100MB")
+    LOGGING_BACKLOG = os.getenv("LOGGING_BACKLOG", 5)
+    LOGGING_FILE_NAME = os.getenv("LOGGING_FILE_NAME", "./log/update_populate.log")
+
+    MONGO_HOST = os.getenv("MONGO_HOST", "localhost")
+    MONGO_PORT = os.getenv("MONGO_PORT", 27017)
+    MONGO_DB = os.getenv("MONGO_DB", "cvexdb")
+    MONGO_USER = os.getenv("MONGO_USER", "")
+    MONGO_PASS = os.getenv("MONGO_PASS", "")
+
+    REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
+    REDIS_PORT = os.getenv("REDIS_PORT", 6379)
+    REDIS_PASS = os.getenv("REDIS_PASS", None)
+    REDIS_Q = os.getenv("REDIS_Q", 9)
+
+    @classmethod
+    def getCVEStartYear(cls):
+        next_year = datetime.datetime.now().year + 1
+        start_year = cls.CVE_START_YEAR
+        if start_year < cls.CVE_START_YEAR or start_year > next_year:
+            print(
+                "The year %i is not a valid year.\ndefault year %i will be used."
+                % (start_year, cls.default["CVEStartYear"])
+            )
+            start_year = cls.default["CVEStartYear"]
+        return start_year
+
+    @classmethod
+    def getProxy(cls):
+        return cls.HTTP_PROXY
+
+    @classmethod
+    def getFeedURL(cls, source):
+        return cls.SOURCES[source]
+
+    @classmethod
+    def toPath(cls, path):
+        return path if os.path.isabs(path) else os.path.join(runPath, "..", path)
+
+    @classmethod
+    def getUpdateLogFile(cls):
+        return cls.toPath(cls.LOGGING_FILE_NAME)
+
+    @classmethod
+    def getMaxLogSize(cls):
+        size = cls.LOGGING_MAX_FILE_SIZE
+        split = re.findall("\d+|\D+", size)
+        multipliers = {"KB": 1024, "MB": 1024 * 1024, "GB": 1024 * 1024 * 1024}
+        if len(split) == 2:
+            base = int(split[0])
+            unit = split[1].strip().upper()
+            return base * multipliers.get(unit, 1024 * 1024)
+        # if size is not a correctly defined set it to 100MB
+        else:
+            return 100 * 1024 * 1024
+
+    @classmethod
+    def getBacklog(cls):
+        return cls.LOGGING_BACKLOG
+
+    @classmethod
+    def getMongoConnection(cls):
+        mongoHost = cls.MONGO_HOST
+        mongoPort = cls.MONGO_PORT
+        mongoDB = cls.MONGO_DB
+        mongoUsername = urllib.parse.quote(cls.MONGO_USER)
+        mongoPassword = urllib.parse.quote(cls.MONGO_PASS)
+        if mongoUsername and mongoPassword:
+            mongoURI = "mongodb://{username}:{password}@{host}:{port}/{db}".format(
+                username=mongoUsername,
+                password=mongoPassword,
+                host=mongoHost,
+                port=mongoPort,
+                db=mongoDB,
+            )
+        else:
+            mongoURI = "mongodb://{host}:{port}/{db}".format(
+                host=mongoHost, port=mongoPort, db=mongoDB
+            )
+        connect = pymongo.MongoClient(mongoURI, connect=False)
+        return connect[mongoDB]
+
+    @classmethod
+    def getRedisQConnection(cls):
+        redisHost = cls.REDIS_HOST
+        redisPort = cls.REDIS_PORT
+        redisDB = cls.REDIS_Q
+        redisPass = cls.REDIS_PASS
+        return redis.Redis(
+            host=redisHost,
+            port=redisPort,
+            db=redisDB,
+            password=redisPass,
+            charset="utf-8",
+            decode_responses=True,
+        )

CveXplore/update/DatabaseLayer.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Database layer translates database calls to functions
+#
+# Software is free software released under the "GNU Affero General Public License v3.0"
+#
+# Copyright (c) 2015-2018  Pieter-Jan Moreels - [email protected]
+
+# imports
+
+import pymongo
+
+from .Config import Configuration as conf
+
+# Variables
+db = conf.getMongoConnection()
+colCVE = db["cves"]
+colCPE = db["cpe"]
+colCWE = db["cwe"]
+colCPEOTHER = db["cpeother"]
+colINFO = db["info"]
+colVIA4 = db["via4"]
+colCAPEC = db["capec"]
+
+mongo_version = db.command("buildinfo")["versionArray"]
+# to check if mongodb > 4.4
+# if it is, then use allow_disk_use for optimized queries
+# to be removed in future with the conditional statements
+# and use allow_disk_use by default
+
+# Functions
+def sanitize(x):
+    if type(x) == pymongo.cursor.Cursor:
+        x = list(x)
+    if type(x) == list:
+        for y in x:
+            sanitize(y)
+    if x and "_id" in x:
+        x.pop("_id")
+    return x
+
+
+# DB Functions
+def ensureIndex(collection, field, **kwargs):
+    db[collection].create_index(field, **kwargs)
+
+
+def drop(collection):
+    db[collection].drop()
+
+
+def setColUpdate(collection, date):
+    colINFO.update({"db": collection}, {"$set": {"last-modified": date}}, upsert=True)
+
+
+def setColInfo(collection, field, data):
+    colINFO.update({"db": collection}, {"$set": {field: data}}, upsert=True)
+
+
+def updateCVE(cve):
+    if cve["cvss3"] is not None:
+        colCVE.update(
+            {"id": cve["id"]},
+            {
+                "$set": {
+                    "cvss3": cve["cvss3"],
+                    "impact3": cve["impact3"],
+                    "exploitability3": cve["exploitability3"],
+                    "cvss3-vector": cve["cvss3-vector"],
+                    "impactScore3": cve["impactScore3"],
+                    "exploitabilityScore3": cve["exploitabilityScore3"],
+                    "cvss": cve["cvss"],
+                    "summary": cve["summary"],
+                    "references": cve["references"],
+                    "impact": cve["impact"],
+                    "vulnerable_product": cve["vulnerable_product"],
+                    "access": cve["access"],
+                    "cwe": cve["cwe"],
+                    "vulnerable_configuration": cve["vulnerable_configuration"],
+                    "vulnerable_configuration_cpe_2_2": cve[
+                        "vulnerable_configuration_cpe_2_2"
+                    ],
+                    "last-modified": cve["Modified"],
+                }
+            },
+            upsert=True,
+        )
+    else:
+        colCVE.update(
+            {"id": cve["id"]},
+            {
+                "$set": {
+                    "cvss3": cve["cvss3"],
+                    "cvss": cve["cvss"],
+                    "summary": cve["summary"],
+                    "references": cve["references"],
+                    "impact": cve["impact"],
+                    "vulnerable_product": cve["vulnerable_product"],
+                    "access": cve["access"],
+                    "cwe": cve["cwe"],
+                    "vulnerable_configuration": cve["vulnerable_configuration"],
+                    "vulnerable_configuration_cpe_2_2": cve[
+                        "vulnerable_configuration_cpe_2_2"
+                    ],
+                    "last-modified": cve["Modified"],
+                }
+            },
+            upsert=True,
+        )
+
+
+def dropCollection(col):
+    return db[col].drop()
+    # jdt_NOTE: is exactly the same as drop(collection)
+    # jdt_NOTE: use only one of them
+
+
+def getTableNames():
+    # return db.collection_names()
+    # jdt_NOTE: collection_names() is depreated, list_collection_names() should be used instead
+    return db.list_collection_names()
+
+
+def getCPEVersionInformation(query):
+    return sanitize(colCPE.find_one(query))
+
+
+def getCPEs():
+    return sanitize(colCPE.find())
+
+
+def getInfo(collection):
+    return sanitize(colINFO.find_one({"db": collection}))