Database/integration

README.md

@@ -19,6 +19,43 @@ Sublert is a security and reconnaissance tool that was written in Python to leve
 Please refer to below article for a detailed technical explanation:
 - https://medium.com/@yassineaboukir/automated-monitoring-of-subdomains-for-fun-and-profit-release-of-sublert-634cfc5d7708
 
+## Database Configuration (Postgresql)
+
+Install Postgresql database (Debian based systems)
+
+```
+    $ sudo apt-get install postgresql
+```
+
+Create the database user for sublert
+
+```
+    $ sudo -u postgres createuser <username>
+```
+
+Create database for sublert
+
+```
+    $ sudo -u postgres createdb <dbname>
+```
+
+Setup a password for the database user
+
+```
+    $ sudo -u postgres psql
+    psql=# alter user <username> with encrypted password '<password>';
+```
+
+Grant permissions to the database for the user
+
+```
+    $ sudo -u postgres psql
+    psql=# grant all privileges on database <dbname> to <username> ;
+```
+
+Finally update the config.py file to reflect the credentials created above
+
+
 ## Usage
 
 Short Form    | Long Form     | Description
@@ -33,7 +70,7 @@ Short Form    | Long Form     | Description
 
 ## Is there a roadmap?
 YES! The tool is now open sourced to be used by the community but contributions are valuable and highly appreciated. I have a number of items that I will be working on to polish the tool, among of which are:
-- Use of a relational database instead of text files for storage.
+- ~~Use of a relational database instead of text files for storage.~~ (Done)
 - Extracting as much information as possible including: title, status code, screenshot and checking for potential subdomain takeovers.
 - Integrate Telegram too for notification pushes.
 

config.py

@@ -1,13 +1,26 @@
 #!/usr/bin/python
 
+import json
+
+# Path to json file with the configuration data
+with open('/opt/progs/sublert/config_live.json') as data:
+    conf = json.load(data)
+
 # Slack webhooks for notifications
-posting_webhook = "https://hooks.slack.com/services/<secret>"
-errorlogging_webhook = "https://hooks.slack.com/services/<secret>"
-slack_sleep_enabled = True  # bypass Slack rate limit when using free workplace, switch to False if you're using Pro/Ent version.
-at_channel_enabled = True   # Add @channel notifications to Slack messages, switch to False if you don't want to use @channel
+posting_webhook = conf['SLACK']['posting_webhook']
+errorlogging_webhook = conf['SLACK']['errorlogging_webhook']
+slack_sleep_enabled = conf['SLACK']['slack_sleep_enabled']  # bypass Slack rate limit when using free workplace, switch to False if you're using Pro/Ent version.
+at_channel_enabled = conf['SLACK']['at_channel_enabled']   # Add @channel notifications to Slack messages, switch to False if you don't want to use @channel
 
 # crtsh postgres credentials, please leave it unchanged.
-DB_HOST = 'crt.sh'
-DB_NAME = 'certwatch'
-DB_USER = 'guest'
-DB_PASSWORD = ''
+DB_HOST = conf['CRTSH']['db_host']
+DB_NAME = conf['CRTSH']['db_name']
+DB_USER = conf['CRTSH']['db_user']
+DB_PASSWORD = conf['CRTSH']['db_password']
+
+# sublert postgresql database credentials
+sldb_user = conf['SLDB']['uname']
+sldb_pass = conf['SLDB']['pass']
+sldb_host = conf['SLDB']['host']
+sldb_dbname = conf['SLDB']['dbname']
+sldb_port = conf['SLDB']['port']

db/Models.py

@@ -0,0 +1,34 @@
+# Database intergration by s-raza (@pyrod)
+from sqlalchemy import *
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import ForeignKey
+from sqlalchemy.orm import relationship
+
+
+Base = declarative_base()
+
+class Domain(Base):
+
+    __tablename__ = "domains"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(String(127), unique=True)
+    date_added = Column(DateTime, default=func.now())
+    date_updated = Column(DateTime)
+
+
+
+class SubDomain(Base):
+
+    __tablename__ = "subdomains"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(String(127), unique=True)
+    date_added = Column(DateTime, default=func.now())
+
+
+    domain_id = Column(Integer, ForeignKey('domains.id'))
+
+    domain = relationship("Domain", back_populates="subdomains")
+
+Domain.subdomains = relationship("SubDomain", back_populates="domain",  cascade = "all, delete, delete-orphan")

db/SLDB.py

@@ -0,0 +1,152 @@
+# Database intergration by s-raza (@pyrod)
+from db.Models import *
+from sqlalchemy import create_engine
+from sqlalchemy.orm import scoped_session, sessionmaker
+#import sublert.config as cfg
+
+class SLDB():
+
+    def __init__(self, *args, **kwargs):
+
+
+        self.db_uname = kwargs['db_uname'] if (kwargs.get('db_uname') is not None) else "sublert"
+        self.db_pass = kwargs['db_pass'] if (kwargs.get('db_pass') is not None) else "sublertpass"
+        self.db_host = kwargs['db_host'] if (kwargs.get('db_host') is not None) else "localhost"
+        self.db_name = kwargs['db_name'] if (kwargs.get('db_name') is not None) else "sublertdb"
+        self.db_port = kwargs['db_port'] if (kwargs.get('db_port') is not None) else "5432"
+
+        self.conn_str = "postgresql://{}:{}@{}:{}/{}".format(self.db_uname, self.db_pass, self.db_host, self.db_port, self.db_name)
+
+        self.engine = create_engine(self.conn_str)
+
+        self.base = Base
+
+        self.base.metadata.create_all(self.engine)
+
+        self.session = scoped_session( sessionmaker(bind=self.engine) )
+
+        self.domain = kwargs['domain'] if (kwargs.get('domain') is not None) else None
+
+
+    def __get_domain_inst(self, domain_name):
+
+        '''Get the instance of a domain from the database.'''
+
+        return self.session.query(Domain).filter_by(name=domain_name).first()
+
+
+
+    def __get_or_create_domain_inst(self, domain_name):
+
+        '''Get the instance of a domain from the database, create it if it does not exist by name.'''
+
+        domain_inst = self.session.query(Domain).filter_by(name=domain_name).first()
+
+
+
+        if domain_inst is None:
+
+            new_domain = self.add_domain(domain_name=domain_name)
+
+            return new_domain
+
+        else:
+
+            return domain_inst       
+
+    def add_domain(self, domain_name):
+        '''Insert a new domain in the database'''
+
+        inst = Domain(name=domain_name)
+
+        self.session.add(inst)
+
+        self.session.commit()
+
+        return inst
+
+    def domain_exists(self, domain_name):
+        '''Check if a domain already exists in the database'''
+
+        domain_inst = self.session.query(Domain).filter_by(name=domain_name).first()
+
+        return False if domain_inst is None else True
+
+    def subdomain_exists(self, subdomain_name):
+        '''Check if a subdomain already exists in the database'''
+
+        subdomain_inst = self.session.query(SubDomain).filter_by(name=subdomain_name).first()
+
+        return False if subdomain_inst is None else True        
+
+    def get_all_subdomains(self, domain):
+        '''Return a list of all the subdomains for a given domain.'''
+
+        domain_inst =  self.__get_domain_inst(domain)
+
+        return [s.name for s in domain_inst.subdomains]
+
+    def get_all_domains(self):
+        '''Return a list of all the domains.'''
+
+        return [s.name for s in self.session.query(Domain).all()]
+
+    def insert_subdomains(self, domain_name, subdomains):
+        '''Insert a list of subdomains or a single subdomain into the database for a given domain'''
+
+
+        domain_inst = self.__get_domain_inst(domain_name)
+
+        domain_inst.date_updated = func.now()
+
+        if type(subdomains) == type([]):
+
+            for sub in subdomains:
+
+                domain_inst.subdomains.append(SubDomain(name=sub, date_added=func.now()))
+
+        else:
+
+            domain_inst.subdomains.append(SubDomain(name=subdomains, date_added=func.now()))
+
+        self.session.commit()
+
+        return domain_inst
+
+
+
+    def delete_domain(self, domain):
+        '''Delete a domain'''
+
+        domain_inst = self.__get_domain_inst(domain)
+
+        self.session.delete(domain_inst)
+
+        self.session.commit()
+
+    def delete_all_domains(self):
+        '''Delete all domains'''
+
+        domain_inst = self.session.query(Domain).all()
+
+
+        for domain in domain_inst:
+
+            self.session.delete(domain)
+
+        self.session.commit()
+
+    def delete_all_subdomains(self, domain):
+        '''Delete all subdomains for a given domain'''
+
+        domain_inst = self.__get_domain_inst(domain)
+
+
+        for subdomain in domain_inst.subdomains:
+
+            self.session.delete(subdomain)
+
+        self.session.commit()         
+
+
+

requirements.txt

@@ -3,3 +3,4 @@ tld
 requests
 dnspython
 psycopg2-binary
+sqlalchemy