Updating from the Vulture Project internal developpement Git

CMakeLists.txt

@@ -3,15 +3,12 @@ project(mod_defender)
 set(CMAKE_BUILD_TYPE Release)
 set(CMAKE_SHARED_LIBRARY_PREFIX "")
 
-include_directories(deps/
-        /usr/include/apache2
-        /usr/include/apr-1.0
-        /usr/include/apr-1
-        /usr/include
-        /usr/local/include/apr-1/
-        /usr/local/include/
-        /usr/local/include/apache24
-        /usr/home/vlt-sys/Engine/include/)
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules")
+
+find_package(Apache)
+find_package(Apr)
+
+include_directories(deps ${APACHE_INC} ${APR_INC})
 
 set(CMAKE_CXX_STANDARD 11)
 

README.md

@@ -12,6 +12,11 @@ It uses the same configs format and is thus fully compatible with [NXAPI/NXTOOL]
     - [Learning log](https://github.com/nbs-system/naxsi/wiki/naxsilogs#naxsi_fmt)
     - [Extensive learning log](https://github.com/nbs-system/naxsi/wiki/naxsilogs#naxsi_exlog)
 
+## Advantages
+  - Human readable log: colored output to watch Mainrules and Basicrules processing
+  - JSON match log: easier parsing and more compact logs
+  - Combined log: regular and extensive match log are mixed so that content and name of variable in question are presents on the same line
+
 ## Required packages
 * apache2 dev package to provide Apache2 headers
 * apr package to provide Apache Portal Runtime library and headers

RuleParser.cpp

@@ -597,16 +597,18 @@ bool RuleParser::isRuleWhitelisted(const http_rule_t &rule, const string &uri, c
     /* Check if the rule is part of disabled rules for this location */
     for (const http_rule_t &disabledRule : disabled_rules) {
         if (checkIds(rule.id, disabledRule.wlIds)) { // Is rule disabled ?
-            if (!disabledRule.br.active) { // if it doesn't specify zone, skip zone-check
-                continue;
-            }
-
             /* If rule target nothing, it's whitelisted everywhere */
             if (!(disabledRule.br.argsMz || disabledRule.br.headersMz ||
                   disabledRule.br.bodyMz || disabledRule.br.urlMz)) {
+                DEBUG_CONF_WL("rule " << rule.id << " not targeting any zone, whitelisted everywhere");
                 return true;
             }
 
+            if (!disabledRule.br.active) { // if it doesn't specify zone, skip zone-check
+                DEBUG_CONF_WL("rule " << rule.id << " not targeting any zone, skipping zone-check");
+                continue;
+            }
+
             /* if exc is in name, but rule is not specificaly disabled for name (and targets a zone)  */
             if (targetName != disabledRule.br.targetName)
                 continue;

RuntimeScanner.cpp

@@ -38,22 +38,28 @@ void RuntimeScanner::applyRuleMatch(const http_rule_t &rule, unsigned long nbMat
 
     if (rulesMatchedCount > 0)
         matchVars << "&";
-    matchVars << "zone" << rulesMatchedCount << "=" << match_zones[zone] << "&";
+    matchVars << "zone" << rulesMatchedCount << "=" << match_zones[zone] << (targetName ? "|NAME" : "") << "&";
     matchVars << "id" << rulesMatchedCount << "=" << rule.id << "&";
     matchVars << "var_name" << rulesMatchedCount << "=" << name;
 
     if (learningJSONLogFile) {
-        if (rulesMatchedCount > 0)
-            jsonMatchVars << ",";
-        jsonMatchVars << "{\"zone\":\"" << match_zones[zone] << "\",";
-        jsonMatchVars << "\"id\":" << rule.id << ",";
-        jsonMatchVars << "\"var_name\":\"" << escapeQuotes(name) << "\"";
-        if (extensiveLearning)
-            jsonMatchVars << ",\"content\":\"" << escapeQuotes(value) << "\"";
-        jsonMatchVars << "}";
+        string fullZone = match_zones[zone];
+        fullZone += (targetName ? "|NAME" : "");
+        string matchInfoKey = name + "#" + fullZone;
+
+        if (matchInfos.find(matchInfoKey) != matchInfos.end()) {
+            matchInfos[matchInfoKey].ruleId.insert(rule.id);
+        } else {
+            matchInfos[matchInfoKey].zone = fullZone;
+            matchInfos[matchInfoKey].ruleId.insert(rule.id);
+            matchInfos[matchInfoKey].varname = name;
+            if (extensiveLearning && !targetName)
+                matchInfos[matchInfoKey].content = value;
+        }
     }
 
-    writeExtensiveLog(rule, zone, name, value, targetName);
+    if (extensiveLearning)
+        writeExtensiveLog(rule, zone, name, value, targetName);
 
     rulesMatchedCount++;
 }
@@ -727,25 +733,25 @@ int RuntimeScanner::processBody() {
 
     /* Stop if BODY size exceeded the limit */
     if (bodyLimitExceeded) {
-        applyRuleMatch(parser.bigRequest, 1, BODY, empty, empty, false);
+        applyCheckRule(parser.bigRequest, 1, empty, empty, BODY, false);
         return processAction();
     }
 
     /* Stop if BODY is empty */
     if (body.empty()) {
-        applyRuleMatch(parser.emptyPostBody, 1, BODY, empty, empty, false);
+        applyCheckRule(parser.emptyPostBody, 1, empty, empty, BODY, false);
         return processAction();
     }
 
     if (contentType == CONTENT_TYPE_UNSUPPORTED) {
-        applyRuleMatch(parser.uncommonContentType, 1, HEADERS, empty, empty, false);
+        applyCheckRule(parser.uncommonContentType, 1, empty, empty, HEADERS, false);
         return processAction();
     }
 
     /* If Content-Type: application/x-www-form-urlencoded */
     if (contentType == CONTENT_TYPE_URL_ENC) {
         if (splitUrlEncodedRuleset(&body[0], bodyRules, BODY)) {
-            applyRuleMatch(parser.uncommonUrl, 1, BODY, empty, empty, false);
+            applyCheckRule(parser.uncommonUrl, 1, empty, empty, BODY, false);
         }
     }
         /* If Content-Type: multipart/form-data */
@@ -847,8 +853,6 @@ void RuntimeScanner::writeLearningLog() {
 
 void RuntimeScanner::writeExtensiveLog(const http_rule_t &rule, MATCH_ZONE zone, const string &name,
                                        const string &value, bool targetName) {
-    if (!extensiveLearning)
-        return;
     stringstream extensivelog;
     extensivelog << naxsiTimeFmt() << " ";
     extensivelog << "[error] ";
@@ -886,7 +890,7 @@ void RuntimeScanner::writeJSONLearningLog() {
     stringstream jsonlog;
     std::time_t result = std::time(nullptr);
     std::asctime(std::localtime(&result));
-    jsonlog << "{\"timestamp\":";
+    jsonlog << "{\"time\":";
     jsonlog << result << ",";
 
     jsonlog << "\"ip\":\"" << clientIp << "\",";
@@ -895,15 +899,33 @@ void RuntimeScanner::writeJSONLearningLog() {
 
     jsonlog << "\"block\":" << (block || drop) << ",";
     jsonlog << "\"scores\":{";
-    int i = 0;
     for (const auto &match : matchScores) {
-        jsonlog << "\"" << match.first << "\":" << match.second << ",";
-        i++;
+        string scoreName = match.first.substr(1, match.first.length() - 1);
+        transform(scoreName.begin(), scoreName.end(), scoreName.begin(), tolower);
+        jsonlog << "\"" << scoreName << "\":" << match.second << ",";
     }
-    if (i > 0) jsonlog.seekp(-1, std::ios_base::end);
-    jsonlog << "},";
-
-    jsonlog << "\"vars\":[" << jsonMatchVars.str() << "],";
+    if (matchScores.size() > 0) jsonlog.seekp(-1, std::ios_base::end);
+
+    jsonlog << "},\"match\":[";
+    for (const auto &matchInfoPair : matchInfos) {
+        const match_info_t &matchInfo = matchInfoPair.second;
+        jsonlog << "{\"zone\":\"" << matchInfo.zone << "\",";
+
+        jsonlog << "\"id\":[";
+        for (const unsigned long &ruleId : matchInfo.ruleId)
+            jsonlog << ruleId << ",";
+        jsonlog.seekp(-1, std::ios_base::end);
+        jsonlog << "]";
+
+        if (!matchInfo.varname.empty())
+            jsonlog << ",\"var_name\":\"" << escapeQuotes(matchInfo.varname) << "\"";
+        if (extensiveLearning && !matchInfo.content.empty())
+            jsonlog << ",\"content\":\"" << escapeQuotes(matchInfo.content) << "\"";
+        jsonlog << "},";
+    }
+
+    if (matchInfos.size() > 0) jsonlog.seekp(-1, std::ios_base::end);
+    jsonlog << "],";
 
     jsonlog << "\"client\":\"" << clientIp << "\",";
     jsonlog << "\"server\":\"" << serverHostname << "\",";

RuntimeScanner.hpp

@@ -13,6 +13,7 @@
 
 #include <map>
 #include <vector>
+#include <set>
 #include <algorithm>
 #include <iostream>
 #include <sstream>
@@ -44,6 +45,7 @@ using namespace Util;
 using std::pair;
 using std::make_pair;
 using std::vector;
+using std::set;
 using std::string;
 using std::cerr;
 using std::stringstream;
@@ -90,12 +92,18 @@ enum LOG_LVL {
     LOG_LVL_DEBUG
 };
 
+typedef struct {
+    string zone;
+    set<unsigned long> ruleId;
+    string varname;
+    string content;
+} match_info_t; 
+
 class RuntimeScanner {
     friend class JsonValidator;
 private:
     RuleParser& parser;
     stringstream matchVars;
-    stringstream jsonMatchVars;
     unsigned int rulesMatchedCount = 0;
     string uri;
     vector<pair<string, string>> headers;
@@ -134,6 +142,7 @@ class RuntimeScanner {
     bool libinjXSS;
 
     unordered_map<string, int> matchScores;
+    unordered_map<string, match_info_t> matchInfos;
 
     bool block = false;
     bool drop = false;

cmake/modules/FindApache.cmake

@@ -0,0 +1,11 @@
+find_path(APACHE_INC
+        NAMES httpd.h
+        HINTS
+        /usr/include/apache2
+        /usr/include
+        /usr/local/include/apache2
+        /usr/local/include/apache22
+        /usr/local/include/apache24
+        /usr/home/vlt-sys/Engine/include)
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(APACHE DEFAULT_MSG APACHE_INC)

cmake/modules/FindApr.cmake

@@ -0,0 +1,9 @@
+find_path(APR_INC
+        NAMES apr.h
+        HINTS
+        /usr/include/apr-1
+        /usr/include/apr-1.0
+        /usr/local/include/apr-1
+        /usr/local/include/apr-1.0)
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(APR DEFAULT_MSG APR_INC)

mod_defender.cpp

@@ -116,12 +116,14 @@ static int post_config(apr_pool_t *pconf, apr_pool_t *, apr_pool_t *, server_rec
 }
 
 static int pass_in_env(request_rec *r, RuntimeScanner *scanner) {
-    if ((scanner->block && !scanner->learning) || scanner->drop)
-        apr_table_set(r->subprocess_env, "defender_action", "block");
-    for (const auto &match : scanner->matchScores) {
-        apr_table_set(r->subprocess_env, apr_psprintf(r->pool, "defender_%s", match.first.c_str()),
-                      apr_itoa(r->pool, match.second));
+    // if ((scanner->block && !scanner->learning) || scanner->drop) {  // NOT USING BLOCK OR DROP IN VULTURE
+    if (!scanner->learning) {
+        for (const auto &match : scanner->matchScores) {
+            apr_table_set(r->subprocess_env, apr_psprintf(r->pool, "defender_%s", match.first.c_str()),
+                          apr_itoa(r->pool, match.second));
+        }
     }
+
     return DECLINED;
 }