first commit

LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2017, Marcin Dulak
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,21 @@
+# Description
+
+An example of `snort++` (https://www.snort.org/snort3) network Intrusion Detection and Prevention System (ID/IPS) deployed on an endpoint Apache host.
+
+The setup combines Vagrant (https://www.vagrantup.com) with Jupyter (http://jupyter.org/) in order to
+achieve a "reproducible", executable documentation in the spirit of https://en.wikipedia.org/wiki/Literate_programming
+
+Please go to [vagrant-snort-nfqueue-tutorial-centos7.ipynb](vagrant-snort-nfqueue-tutorial-centos7.ipynb)
+
+
+# Dependencies
+
+None
+
+
+# License
+
+BSD 2-clause
+
+
+# Todo

Vagrantfile

@@ -0,0 +1,101 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+NETWORK = ENV.fetch('NETWORK', '192.168.17.')
+
+hosts = {
+  'attacker0' => {'hostname' => 'attacker0', 'ip' => NETWORK + '10', 'mac' => '080027001710'},
+  'monitor0' => {'hostname' => 'monitor0', 'ip' => NETWORK + '20', 'mac' => '080027001720',
+                 'prometheus_port' => {'guest' => 9090, 'host' => 9090}},
+  'snort0' => {'hostname' => 'snort0', 'ip' => NETWORK + '30', 'mac' => '080027001730'},
+}
+
+Vagrant.configure(2) do |config|
+  hosts.keys.sort.each do |host|
+      config.vm.define hosts[host]['hostname'] do |machine|
+        #machine.vm.box = 'centos/7'
+        machine.vm.box = 'bento/centos-7.3'
+        machine.vm.box_url = machine.vm.box
+        #machine.vm.synced_folder '.', '/vagrant', disabled: true
+        machine.vm.network 'private_network', ip: hosts[host]['ip'], mac: hosts[host]['mac']
+        #if host.start_with?("monitor")
+        #  machine.vm.network 'forwarded_port', guest: hosts[host]['prometheus_port']['guest'], host: hosts[host]['prometheus_port']['host']
+        #end
+        machine.vm.provider 'virtualbox' do |v|
+          if host.start_with?("attacker")
+            v.memory = 2048  # memory needed for tcpreplay caching
+          else
+            v.memory = 512
+          end
+          if host.start_with?("snort")
+            v.cpus = 4  # cores used by snort's multithreading
+          else
+            v.cpus = 1
+          end
+          # disable VBox time synchronization and use ntp
+          v.customize ['setextradata', :id, 'VBoxInternal/Devices/VMMDev/0/Config/GetHostTimeDisabled', 1]
+        end
+      end
+  end
+  # disable IPv6 on Linux
+  $linux_disable_ipv6 = <<SCRIPT
+sysctl -w net.ipv6.conf.default.disable_ipv6=1
+sysctl -w net.ipv6.conf.all.disable_ipv6=1
+sysctl -w net.ipv6.conf.lo.disable_ipv6=1
+SCRIPT
+  # setenforce 0
+  $setenforce_0 = <<SCRIPT
+if test `getenforce` = 'Enforcing'; then setenforce 0; fi
+#sed -Ei 's/^SELINUX=.*/SELINUX=Permissive/' /etc/selinux/config
+SCRIPT
+  # stop firewalld
+  $systemctl_stop_firewalld = <<SCRIPT
+systemctl stop firewalld.service
+SCRIPT
+  # common settings on all machines
+  $etc_hosts = <<SCRIPT
+echo "$*" >> /etc/hosts
+SCRIPT
+  $epel7 = <<SCRIPT
+yum -d 1 -e 0 -y install https://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-9.noarch.rpm
+SCRIPT
+  $snort_el = <<SCRIPT
+cat <<'END' > /etc/yum.repos.d/copr-marcindulak-snort.repo
+[copr-marcindulak-snort]
+name=copr-marcindulak-snort
+baseurl=https://copr-be.cloud.fedoraproject.org/results/marcindulak/snort/epel-$releasever-$basearch
+enabled=1
+gpgcheck=1
+gpgkey=https://copr-be.cloud.fedoraproject.org/results/marcindulak/snort/pubkey.gpg
+END
+SCRIPT
+  # https://github.com/lest/prometheus-rpm
+  $prometheus_el = <<SCRIPT
+curl -s https://packagecloud.io/install/repositories/prometheus-rpm/release/script.rpm.sh | bash
+SCRIPT
+  hosts.keys.sort.each do |host|
+    config.vm.define hosts[host]['hostname'] do |machine|
+      machine.vm.provision :shell, :inline => 'hostname ' + hosts[host]['hostname'], run: 'always'
+      machine.vm.provision :shell, :inline => 'echo ' + hosts[host]['hostname'] + ' > /etc/hostname'
+      hosts.keys.sort.each do |k|
+        machine.vm.provision 'shell' do |s|
+          s.inline = $etc_hosts
+          s.args   = [hosts[k]['ip'], hosts[k]['hostname']]
+        end
+      end
+      machine.vm.provision :shell, :inline => $setenforce_0, run: 'always'
+      machine.vm.provision :shell, :inline => $epel7
+      if host.start_with?("snort")
+        machine.vm.provision :shell, :inline => $snort_el
+      end
+      if host.start_with?("monitor") or host.start_with?("snort")
+        machine.vm.provision :shell, :inline => $prometheus_el
+      end
+      machine.vm.provision :shell, :inline => $linux_disable_ipv6, run: 'always'
+      # install and enable ntp
+      machine.vm.provision :shell, :inline => 'yum -d 1 -e 0 -y install ntp'
+      machine.vm.provision :shell, :inline => 'systemctl enable ntpd'
+      machine.vm.provision :shell, :inline => 'systemctl start ntpd'
+    end
+  end
+end

configure_node_exporter_on_snort.sh

@@ -0,0 +1,7 @@
+# https://github.com/lest/prometheus-rpm
+sudo su - -c "curl -s https://packagecloud.io/install/repositories/prometheus-rpm/release/script.rpm.sh | sed 's/yum install -y/yum install -d0 -e0 -y/' | bash"
+sudo su - -c "yum -d 1 -e 0 -y install node_exporter"
+# configure node_exporter
+sudo su - -c "echo NODE_EXPORTER_OPTS='-collector.textfile.directory=/var/lib/prometheus' >> /etc/default/node_exporter"
+sudo su - -c "systemctl enable node_exporter"
+sudo su - -c "systemctl start node_exporter"

configure_prometheus_on_monitor.sh

@@ -0,0 +1,8 @@
+# https://github.com/lest/prometheus-rpm
+sudo su - -c "curl -s https://packagecloud.io/install/repositories/prometheus-rpm/release/script.rpm.sh | sed 's/yum install -y/yum install -d0 -e0 -y/' | bash"
+sudo su - -c "yum -d 1 -e 0 -y install prometheus"
+# configure prometheus to scrape node_exporter
+sudo su - -c """sed -i '/The job name is added as a label/a\ \n  # Metrics from nodes using \"node_exporter\".\n  - job_name: \"node_exporter\"\n    metrics_path: \"/metrics\"\n    file_sd_configs:\n     \
+ - files:\n        - /etc/prometheus/node_exporter.json\n' /etc/prometheus/prometheus.yml"""
+sudo su - -c "systemctl enable prometheus"
+sudo su - -c "systemctl start prometheus"

copr-marcindulak-snort.repo

@@ -0,0 +1,6 @@
+[copr-marcindulak-snort]
+name=copr-marcindulak-snort
+baseurl=https://copr-be.cloud.fedoraproject.org/results/marcindulak/snort/epel-$releasever-$basearch
+enabled=1
+gpgcheck=1
+gpgkey=https://copr-be.cloud.fedoraproject.org/results/marcindulak/snort/pubkey.gpg

node_exporter.json

@@ -0,0 +1,12 @@
+[
+    {
+	"labels": {
+	    "job": "node"
+	}, 
+	"targets": [
+	    "snort0:9100",
+	    "snort1:9100"
+	]
+    }
+]
+

pytbull_config.cfg

@@ -0,0 +1,63 @@
+[CLIENT]
+ipaddr                  = 192.168.17.10
+iface                   = enp0s8
+useproxy                = 0
+proxyhost               = 
+proxyport               = 
+proxyuser               = 
+proxypass               = 
+
+[PATHS]
+db                      = data/pytbull.db
+#urlpdf                  = http://dl.dropbox.com/u/30211728/
+urlpdf                  = http://wedontusethat/
+pdfdir                  = pdf/malicious
+pcapdir                 = pcap
+tempfile                = /tmp/pytbull.tmp
+alertsfile              = /var/log/snort/alert_fast.txt
+#alertsfile              = /var/log/suricata/fast.log
+
+[ENV]
+sudo                    = /usr/bin/sudo
+nmap                    = /usr/bin/nmap
+nikto                   = /usr/bin/nikto
+niktoconf               = /etc/nikto/config
+hping3                  = /usr/sbin/hping3
+tcpreplay               = /usr/bin/tcpreplay
+ab                      = /usr/bin/ab
+ping                    = /usr/bin/ping
+ncrack                  = /usr/bin/ncrack
+ncrackusers             = data/ncrack-users.txt
+ncrackpasswords         = data/ncrack-passwords.txt
+localhost               = 127.0.0.1
+
+[FTP]
+ftpproto                = ftp
+ftpport                 = 21
+ftpuser                 = pilou
+ftppasswd               = oopsoops
+
+[TIMING]
+sleepbeforegetalerts    = 2
+sleepbeforenexttest     = 2
+sleepbeforetwoftp       = 2
+urltimeout              = 10
+
+[SERVER]
+reverseshellport        = 12345
+
+[TESTS]
+clientSideAttacks       = 0
+testRules               = 1
+badTraffic              = 1
+fragmentedPackets       = 1
+bruteForce              = 1
+evasionTechniques       = 1
+shellCodes              = 0
+denialOfService         = 1
+pcapReplay              = 0
+normalUsage             = 1
+ipReputation            = 0
+
+[TESTS_PARAMS]
+ipreputationnbtests     = 10

snort-alert-count.py

@@ -0,0 +1,77 @@
+import dateutil.parser
+import json
+import os
+from pwd import getpwnam
+import shutil
+import tempfile
+import time
+
+from idstools import maps
+from idstools.scripts.u2json import load_from_snort_conf
+from idstools.scripts.u2json import Formatter
+from idstools import unified2
+
+import argparse
+import warnings
+
+warnings.filterwarnings("ignore")
+
+def create_parser():
+    parser = argparse.ArgumentParser(
+        description='Write Snort alerts in Prometheus Textfile Collector format',
+    )
+
+    parser.add_argument("--directory", default='/var/log/snort',
+                        help="Spool directory with unified2 logs: default /var/log/snort")
+    parser.add_argument("--prefix", default='unified2.log',
+                        help="Name (prefix) of the unified2 log file: default unified2.log")
+    parser.add_argument("--output", default='/var/lib/prometheus/snort.prom', required=False, help="Full path to output file")
+    parser.add_argument("--node_exporter_username", default='prometheus', required=False, help="node_exporter user name")
+    parser.add_argument("--node_exporter_groupname", default='prometheus', required=False, help="node_exporter group name")
+    parser.add_argument("--timestamp", default=False, action='store_true', help="Include timestamp in prometheus metrics")
+    parser.add_argument("--threshold_seconds", default=20,
+                        required=False, type=int, help="Spool to the output file every threshold_seconds, 20 by default")
+
+    return parser
+
+parser = create_parser()
+args = parser.parse_args()
+
+assert os.path.exists(args.directory), args.directory + ': no such file or directory'
+assert os.path.exists(os.path.dirname(args.output)), os.path.dirname(args.output) + ': no such file or directory'
+
+msgmap = maps.SignatureMap()
+classmap = maps.ClassificationMap()
+formatter = Formatter(msgmap=msgmap, classmap=classmap)
+
+reader = unified2.SpoolRecordReader(directory=args.directory, prefix=args.prefix, follow=True, init_filename=None, init_offset=None)
+
+labels = ['generator-id', 'signature-id', 'blocked', 'source-ip', 'dport-icode']
+
+alert_count = {}
+text_collector = {}
+last_time = time.time()
+for record in reader:
+    formatted = dict(formatter.format(record))
+    if not 'event' in formatted:
+        continue
+    event = formatted['event']
+    #[('event', {'impact': 0, 'protocol': 6, 'classification': 'Misc Attack', 'dport-icode': 443, 'signature-revision': 8, 'classification-id': 30, 'signature-id': 10997, 'sensor-id': 0, 'impact-flag': 0, 'sport-itype': 60748, 'priority': 2, 'event-second': 1481294223, 'generator-id': 1, 'destination-ip': '10.255.2.160', 'event-id': 6, 'vlan-id': None, 'mpls-label': None, 'msg': 'SERVER-WEBAPP SSLv2 OpenSSl KEY_ARG buffer overflow attempt', 'source-ip': '10.255.2.200', 'event-microsecond': 885673, 'blocked': 0})]
+    alert_id = 'snort' + ''.join(['_' + str(event[label]) for label in labels])
+    alert_count[alert_id] = alert_count.get(alert_id, 0) + 1
+    timestamp = int(float(event['event-second']) * 1000 + float(event['event-microsecond']) / 1000)
+    entry = 'snort_alert_count{' + ','.join([key.replace('-', '_') + '=' '"' + str(value) + '"' for key, value in list((key, event[key]) for key in labels)]) + '} '
+    if args.timestamp:
+        text_collector.update({entry: str(alert_count[alert_id]) + ' ' + str(timestamp)})
+    else:
+        text_collector.update({entry: str(alert_count[alert_id])})
+    if time.time() - last_time > int(args.threshold_seconds):
+        with tempfile.NamedTemporaryFile(delete=True) as temp:
+            temp.write('# HELP snort_alert_count Snort alert count.\n')
+            temp.write('# TYPE snort_alert_count counter\n')
+            for key in sorted(text_collector.keys()):
+                temp.write(key + ' ' + text_collector[key] + '\n')
+            temp.flush()
+            shutil.copy2(temp.name, args.output)
+            os.chown(args.output, getpwnam(args.node_exporter_username).pw_uid, getpwnam(args.node_exporter_groupname).pw_gid)
+        last_time = time.time()

snort-alert-count.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=snort-alert-count
+After=syslog.target network.target snort@enp0s8
+Requires=snort@enp0s8
+PartOf=snort@enp0s8
+
+[Service]
+Restart=on-failure
+User=root
+Group=root
+ExecStartPre=/usr/bin/sleep 5
+ExecStart=/bin/sh -c '/usr/bin/python /usr/local/bin/snort-alert-count.py \
+                      --directory /var/log/snort \
+                      --prefix unified2.log \
+		      --output /var/lib/prometheus/snort.prom \
+                      --threshold_seconds=20 2>&1'
+
+[Install]
+WantedBy=multi-user.target

snort-test-alert.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=snort-test-alert
+After=syslog.target network.target
+
+[Service]
+Restart=on-failure
+ExecStart=/bin/sh -c 'while /bin/true; do curl -s -m 1 http://snort0/snort-test-alert > /dev/null 2>&1; sleep 10; done'
+
+[Install]
+WantedBy=multi-user.target