Changes for devel to master for 2.1

* Added scripts to extract other executable mimetypes
* Enable extraction of executable types by default
* Add in the Salesforce JA3 SSL fingerprinting. From salesforce/ja3
* Remove redundant @load call to itself (#13)
* Fixed formatting in extraction scripts

frameworks/files/extraction/plugins/__load__.bro

@@ -1,5 +1,6 @@
 #@load ./extract-all-files.bro
 @load ./extract-common-exploit-types.bro
+@load ./extract-executable-types.bro
 #@load ./extract-java.bro
 #@load ./extract-ms-office.bro
 #@load ./extract-pdf.bro

frameworks/files/extraction/plugins/extract-elf.bro

@@ -0,0 +1,16 @@
+@load ../__load__.bro
+
+module FileExtraction;
+
+const linux_types: set[string] = {
+    "application/x-object",
+    "application/x-executable",
+    "application/x-sharedlib",
+    "application/x-coredump"
+};
+
+hook FileExtraction::extract(f: fa_file, meta: fa_metadata) &priority=5
+{
+  if ( meta$mime_type in linux_types )
+    break;
+}

frameworks/files/extraction/plugins/extract-executable-types.bro

@@ -0,0 +1,7 @@
+@load ../__load__.bro
+
+@load ./extract-pe
+@load ./extract-elf
+@load ./extract-macho
+@load ./extract-scripts
+

frameworks/files/extraction/plugins/extract-macho.bro

@@ -0,0 +1,9 @@
+@load ../__load__.bro
+
+module FileExtraction;
+
+hook FileExtraction::extract(f: fa_file, meta: fa_metadata) &priority=5
+{
+	if ( meta$mime_type == "application/x-mach-o-executable" )
+		break;
+}

frameworks/files/extraction/plugins/extract-scripts.bro

@@ -0,0 +1,21 @@
+@load ../__load__.bro
+
+module FileExtraction;
+
+const script_types: set[string] = {
+    "text/x-shellscript",
+    "text/x-perl",
+    "text/x-ruby",
+    "text/x-python",
+    "text/x-awk",
+    "text/x-tcl",
+    "text/x-lua",
+    #"application/javascript", # Let's skip this one, but listing for completeness
+    "text/x-php"
+};
+
+hook FileExtraction::extract(f: fa_file, meta: fa_metadata) &priority=5
+{
+  if ( meta$mime_type in script_types )
+  	break;
+}

misc/ja3/__load__.bro

@@ -0,0 +1,3 @@
+@load ./ja3.bro
+@load ./intel_ja3.bro
+@load ./ja3s.bro

misc/ja3/intel_ja3.bro

@@ -0,0 +1,24 @@
+# This Bro script adds JA3 to the Bro Intel Framework as Intel::JA3
+#
+# Author: John B. Althouse ([email protected])
+#
+# Copyright (c) 2017, salesforce.com, inc.
+# All rights reserved.
+# Licensed under the BSD 3-Clause license. 
+# For full license text, see LICENSE.txt file in the repo root  or https://opensource.org/licenses/BSD-3-Clause
+
+module Intel;
+
+export {
+    redef enum Intel::Type += { Intel::JA3 };
+}
+
+export {
+    redef enum Intel::Where += { SSL::IN_JA3 };
+}
+
+event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec)
+	{
+	if ( c$ssl?$ja3 )
+	Intel::seen([$indicator=c$ssl$ja3, $indicator_type=Intel::JA3, $conn=c, $where=SSL::IN_JA3]);
+	}

misc/ja3/ja3.bro

@@ -0,0 +1,149 @@
+# This Bro script appends JA3 to ssl.log
+# Version 1.3 (June 2017)
+#
+# Authors: John B. Althouse ([email protected]) & Jeff Atkinson ([email protected])
+#
+# Copyright (c) 2017, salesforce.com, inc.
+# All rights reserved.
+# Licensed under the BSD 3-Clause license. 
+# For full license text, see LICENSE.txt file in the repo root  or https://opensource.org/licenses/BSD-3-Clause
+
+module JA3;
+
+export {
+redef enum Log::ID += { LOG };
+}
+
+type TLSFPStorage: record {
+       client_version:  count &default=0 &log;
+       client_ciphers:  string &default="" &log;
+       extensions:      string &default="" &log;
+       e_curves:        string &default="" &log;
+       ec_point_fmt:    string &default="" &log;
+};
+
+redef record connection += {
+       tlsfp: TLSFPStorage &optional;
+};
+
+redef record SSL::Info += {
+  ja3:            string &optional &log;
+# LOG FIELD VALUES ##
+#  ja3_version:  string &optional &log;
+#  ja3_ciphers:  string &optional &log;
+#  ja3_extensions: string &optional &log;
+#  ja3_ec:         string &optional &log;
+#  ja3_ec_fmt:     string &optional &log;
+};
+
+# Google. https://tools.ietf.org/html/draft-davidben-tls-grease-01
+const grease: set[int] = {
+    2570,
+    6682,
+    10794,
+    14906,
+    19018,
+    23130,
+    27242,
+    31354,
+    35466,
+    39578,
+    43690,
+    47802,
+    51914,
+    56026,
+    60138,
+    64250
+};
+const sep = "-";
+event bro_init() {
+    Log::create_stream(JA3::LOG,[$columns=TLSFPStorage, $path="tlsfp"]);
+}
+
+event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
+{
+if ( ! c?$tlsfp )
+    c$tlsfp=TLSFPStorage();
+    if ( is_orig = T ) {
+        if ( code in grease ) {
+            next;
+        }
+        if ( c$tlsfp$extensions == "" ) {
+            c$tlsfp$extensions = cat(code);
+        }
+        else {
+            c$tlsfp$extensions = string_cat(c$tlsfp$extensions, sep,cat(code));
+        }
+    }
+}
+
+event ssl_extension_ec_point_formats(c: connection, is_orig: bool, point_formats: index_vec)
+{
+if ( !c?$tlsfp )
+    c$tlsfp=TLSFPStorage();
+    if ( is_orig = T ) {
+        for ( i in point_formats ) {
+            if ( point_formats[i] in grease ) {
+            next;
+            }
+            if ( c$tlsfp$ec_point_fmt == "" ) {
+            c$tlsfp$ec_point_fmt += cat(point_formats[i]);
+            }
+            else {
+            c$tlsfp$ec_point_fmt += string_cat(sep,cat(point_formats[i]));
+            }
+        }
+    }
+}
+
+event ssl_extension_elliptic_curves(c: connection, is_orig: bool, curves: index_vec)
+{
+    if ( !c?$tlsfp )
+    c$tlsfp=TLSFPStorage();
+    if ( is_orig = T  ) {
+        for ( i in curves ) {
+            if ( curves[i] in grease ) {
+            next;
+            }
+            if ( c$tlsfp$e_curves == "" ) {
+                c$tlsfp$e_curves += cat(curves[i]);
+            }
+            else {
+                c$tlsfp$e_curves += string_cat(sep,cat(curves[i]));
+            }
+        }
+    }
+}
+
+event ssl_client_hello(c: connection, version: count, possible_ts: time, client_random: string, session_id: string, ciphers: index_vec) &priority=1
+{
+    if ( !c?$tlsfp )
+    c$tlsfp=TLSFPStorage();
+    c$tlsfp$client_version = version;
+    for ( i in ciphers ) {
+        if ( ciphers[i] in grease ) {
+            next;
+        }
+        if ( c$tlsfp$client_ciphers == "" ) { 
+            c$tlsfp$client_ciphers += cat(ciphers[i]);
+        }
+        else {
+            c$tlsfp$client_ciphers += string_cat(sep,cat(ciphers[i]));
+        }
+    }
+    local sep2 = ",";
+    local ja3_string = string_cat(cat(c$tlsfp$client_version),sep2,c$tlsfp$client_ciphers,sep2,c$tlsfp$extensions,sep2,c$tlsfp$e_curves,sep2,c$tlsfp$ec_point_fmt);
+    local tlsfp_1 = md5_hash(ja3_string);
+    c$ssl$ja3 = tlsfp_1;
+
+# LOG FIELD VALUES ##
+#c$ssl$ja3_version = cat(c$tlsfp$client_version);
+#c$ssl$ja3_ciphers = c$tlsfp$client_ciphers;
+#c$ssl$ja3_extensions = c$tlsfp$extensions;
+#c$ssl$ja3_ec = c$tlsfp$e_curves;
+#c$ssl$ja3_ec_fmt = c$tlsfp$ec_point_fmt;
+#
+# FOR DEBUGGING ##
+#print "JA3: "+tlsfp_1+" Fingerprint String: "+ja3_string;
+
+}

misc/ja3/ja3s.bro

@@ -0,0 +1,78 @@
+# This Bro script appends JA3S (JA3 Server) to ssl.log
+# Version 1.0 (August 2018)
+# This builds a fingerprint for the SSL Server Hello packet based on SSL/TLS version, cipher picked, and extensions used. 
+# Designed to be used in conjunction with JA3 to fingerprint SSL communication between clients and servers.
+#
+# Authors: John B. Althouse ([email protected]) Jeff Atkinson ([email protected])
+# Copyright (c) 2018, salesforce.com, inc.
+# All rights reserved.
+# Licensed under the BSD 3-Clause license. 
+# For full license text, see LICENSE.txt file in the repo root  or https://opensource.org/licenses/BSD-3-Clause
+#
+
+
+
+module JA3_Server;
+
+export {
+redef enum Log::ID += { LOG };
+}
+
+type JA3Sstorage: record {
+       server_version:      count &default=0 &log;
+       server_cipher:      count &default=0 &log;
+       server_extensions:   string &default="" &log;
+};
+
+redef record connection += {
+       ja3sfp: JA3Sstorage &optional;
+};
+
+redef record SSL::Info += {
+  ja3s:            string &optional &log;
+# LOG FIELD VALUES #
+#  ja3s_version:  string &optional &log;
+#  ja3s_cipher:  string &optional &log;
+#  ja3s_extensions: string &optional &log;
+};
+
+
+const sep = "-";
+event bro_init() {
+    Log::create_stream(JA3_Server::LOG,[$columns=JA3Sstorage, $path="ja3sfp"]);
+}
+
+event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
+{
+if ( ! c?$ja3sfp )
+    c$ja3sfp=JA3Sstorage();
+    if ( is_orig == F ) { 
+        if ( c$ja3sfp$server_extensions == "" ) {
+            c$ja3sfp$server_extensions = cat(code);
+        }
+        else {
+            c$ja3sfp$server_extensions = string_cat(c$ja3sfp$server_extensions, sep,cat(code));
+        }
+    }
+}
+
+event ssl_server_hello(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count) &priority=1
+{
+    if ( !c?$ja3sfp )
+    c$ja3sfp=JA3Sstorage();
+    c$ja3sfp$server_version = version;
+    c$ja3sfp$server_cipher = cipher;
+    local sep2 = ",";
+    local ja3s_string = string_cat(cat(c$ja3sfp$server_version),sep2,cat(c$ja3sfp$server_cipher),sep2,c$ja3sfp$server_extensions);
+    local ja3sfp_1 = md5_hash(ja3s_string);
+    c$ssl$ja3s = ja3sfp_1;
+
+# LOG FIELD VALUES #
+#c$ssl$ja3s_version = cat(c$ja3sfp$server_version);
+#c$ssl$ja3s_cipher = cat(c$ja3sfp$server_cipher);
+#c$ssl$ja3s_extensions = c$ja3sfp$server_extensions;
+#
+# FOR DEBUGGING #
+#print "JA3S: "+ja3sfp_1+" Fingerprint String: "+ja3s_string;
+
+}

plugins/afpacket.bro

@@ -14,5 +14,4 @@
 
 # Workaround for AF_Packet plugin across multiple interfaces
 # See https://bro-tracker.atlassian.net/browse/BIT-1747 for more info
-@load scripts/rock/plugins/afpacket
 redef AF_Packet::fanout_id = strcmp(getenv("fanout_id"),"") == 0 ? 0 : to_count(getenv("fanout_id"));

rock.bro

@@ -44,6 +44,10 @@ redef FileExtract::default_limit = 1048576000;
 # Add sensor and log meta information to each log
 @load ./frameworks/logging/extension
 
+#== 3rd Party Scripts =================================
+# Add Salesforce's JA3 SSL fingerprinting
+@load ./misc/ja3
+
 ### Sensor specific scripts ######################
 
 # Configure AF_PACKET, if in use