bump libnDPI to e946f49aca13e4447a7d7b2acae6323a4531fb55

* incorporated upstream changes Signed-off-by: Toni Uhlig <[email protected]>
utoni · Jan 25, 2025 · 471ea83 · 471ea83
1 parent 064bd3a
commit 471ea83
Show file tree

Hide file tree

Showing 2,666 changed files with 19,832 additions and 12,070 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -93,7 +93,7 @@ jobs:
             upload: false
             ndpi_min_version: "4.12"
           - compiler: "clang-12"
-            os: "ubuntu-latest"
+            os: "ubuntu-20.04"
             ndpi_build: "-DBUILD_NDPI=ON"
             ndpid_examples: "-DBUILD_EXAMPLES=ON"
             ndpid_gcrypt: "-DNDPI_WITH_GCRYPT=OFF"

diff --git a/examples/c-analysed/c-analysed.c b/examples/c-analysed/c-analysed.c
@@ -170,6 +170,7 @@ static struct
         uint64_t flow_category_antimalware_count;
         uint64_t flow_category_crypto_currency_count;
         uint64_t flow_category_gambling_count;
+        uint64_t flow_category_health_count;
         uint64_t flow_category_unknown_count;
 
         uint64_t flow_confidence_by_port;
@@ -220,15 +221,10 @@ struct global_map
     };
 };
 
-#define ANALYSED_STATS_COUNTER_PTR(member)                                                                             \
-    {                                                                                                                  \
-        .global_stat_inc = &(analysed_statistics.counters.member), NULL                                                \
-    }
+#define ANALYSED_STATS_COUNTER_PTR(member) {.global_stat_inc = &(analysed_statistics.counters.member), NULL}
 #define ANALYSED_STATS_GAUGE_PTR(member)                                                                               \
-    {                                                                                                                  \
-        .global_stat_inc = &(analysed_statistics.gauges[0].member),                                                    \
-        .global_stat_dec = &(analysed_statistics.gauges[1].member)                                                     \
-    }
+    {.global_stat_inc = &(analysed_statistics.gauges[0].member),                                                       \
+     .global_stat_dec = &(analysed_statistics.gauges[1].member)}
 #define ANALYSED_STATS_COUNTER_INC(member) (analysed_statistics.counters.member++)
 #define ANALYSED_STATS_GAUGE_RES(member) (analysed_statistics.gauges[0].member--)
 #define ANALYSED_STATS_GAUGE_INC(member) (analysed_statistics.gauges[0].member++)
@@ -329,6 +325,7 @@ static struct global_map const categories_map[] = {
     {"Antimalware", ANALYSED_STATS_GAUGE_PTR(flow_category_antimalware_count)},
     {"Crypto_Currency", ANALYSED_STATS_GAUGE_PTR(flow_category_crypto_currency_count)},
     {"Gambling", ANALYSED_STATS_GAUGE_PTR(flow_category_gambling_count)},
+    {"Health", ANALYSED_STATS_GAUGE_PTR(flow_category_health_count)},
     {NULL, ANALYSED_STATS_GAUGE_PTR(flow_category_unknown_count)}};
 
 static struct global_map const confidence_map[] = {
@@ -1713,7 +1710,7 @@ static int write_global_flow_stats(void)
                                          ANALYSEDB_FORMAT() ANALYSEDB_FORMAT() ANALYSEDB_FORMAT() ANALYSEDB_FORMAT()
                                              ANALYSEDB_FORMAT() ANALYSEDB_FORMAT() ANALYSEDB_FORMAT() ANALYSEDB_FORMAT()
                                                  ANALYSEDB_FORMAT() ANALYSEDB_FORMAT() ANALYSEDB_FORMAT()
-                                                     ANALYSEDB_FORMAT() ANALYSEDB_FORMAT(),
+                                                     ANALYSEDB_FORMAT() ANALYSEDB_FORMAT() ANALYSEDB_FORMAT(),
 
                  ANALYSEDB_VALUE_GAUGE(flow_category_unspecified_count),
                  ANALYSEDB_VALUE_GAUGE(flow_category_media_count),
@@ -1754,6 +1751,7 @@ static int write_global_flow_stats(void)
                  ANALYSEDB_VALUE_GAUGE(flow_category_antimalware_count),
                  ANALYSEDB_VALUE_GAUGE(flow_category_crypto_currency_count),
                  ANALYSEDB_VALUE_GAUGE(flow_category_gambling_count),
+                 ANALYSEDB_VALUE_GAUGE(flow_category_health_count),
                  ANALYSEDB_VALUE_GAUGE(flow_category_unknown_count));
     CHECK_SNPRINTF_RET(bytes);
 
@@ -1891,6 +1889,7 @@ static int write_global_flow_stats(void)
     ANALYSED_STATS_GAUGE_SUB(flow_category_antimalware_count);
     ANALYSED_STATS_GAUGE_SUB(flow_category_crypto_currency_count);
     ANALYSED_STATS_GAUGE_SUB(flow_category_gambling_count);
+    ANALYSED_STATS_GAUGE_SUB(flow_category_health_count);
     ANALYSED_STATS_GAUGE_SUB(flow_category_unknown_count);
 
     ANALYSED_STATS_GAUGE_SUB(flow_confidence_by_port);

diff --git a/examples/c-collectd/c-collectd.c b/examples/c-collectd/c-collectd.c
@@ -176,6 +176,7 @@ static struct
         uint64_t flow_category_antimalware_count;
         uint64_t flow_category_crypto_currency_count;
         uint64_t flow_category_gambling_count;
+        uint64_t flow_category_health_count;
         uint64_t flow_category_unknown_count;
 
         uint64_t flow_confidence_by_port;
@@ -226,15 +227,10 @@ struct global_map
     };
 };
 
-#define COLLECTD_STATS_COUNTER_PTR(member)                                                                             \
-    {                                                                                                                  \
-        .global_stat_inc = &(collectd_statistics.counters.member), NULL                                                \
-    }
+#define COLLECTD_STATS_COUNTER_PTR(member) {.global_stat_inc = &(collectd_statistics.counters.member), NULL}
 #define COLLECTD_STATS_GAUGE_PTR(member)                                                                               \
-    {                                                                                                                  \
-        .global_stat_inc = &(collectd_statistics.gauges[0].member),                                                    \
-        .global_stat_dec = &(collectd_statistics.gauges[1].member)                                                     \
-    }
+    {.global_stat_inc = &(collectd_statistics.gauges[0].member),                                                       \
+     .global_stat_dec = &(collectd_statistics.gauges[1].member)}
 #define COLLECTD_STATS_COUNTER_INC(member) (collectd_statistics.counters.member++)
 #define COLLECTD_STATS_GAUGE_RES(member) (collectd_statistics.gauges[0].member--)
 #define COLLECTD_STATS_GAUGE_INC(member) (collectd_statistics.gauges[0].member++)
@@ -335,6 +331,7 @@ static struct global_map const categories_map[] = {
     {"Antimalware", COLLECTD_STATS_GAUGE_PTR(flow_category_antimalware_count)},
     {"Crypto_Currency", COLLECTD_STATS_GAUGE_PTR(flow_category_crypto_currency_count)},
     {"Gambling", COLLECTD_STATS_GAUGE_PTR(flow_category_gambling_count)},
+    {"Health", COLLECTD_STATS_GAUGE_PTR(flow_category_health_count)},
     {NULL, COLLECTD_STATS_GAUGE_PTR(flow_category_unknown_count)}};
 
 static struct global_map const confidence_map[] = {
@@ -655,7 +652,7 @@ static void print_collectd_exec_output(void)
                                                        COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT()
                                                            COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT()
                                                                COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT()
-                                                                   COLLECTD_GAUGE_N_FORMAT(),
+                                                                   COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT(),
 
            COLLECTD_GAUGE_N(flow_category_unspecified_count),
            COLLECTD_GAUGE_N(flow_category_media_count),
@@ -696,6 +693,7 @@ static void print_collectd_exec_output(void)
            COLLECTD_GAUGE_N(flow_category_antimalware_count),
            COLLECTD_GAUGE_N(flow_category_crypto_currency_count),
            COLLECTD_GAUGE_N(flow_category_gambling_count),
+           COLLECTD_GAUGE_N(flow_category_health_count),
            COLLECTD_GAUGE_N(flow_category_unknown_count));
 
     printf(COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT() COLLECTD_GAUGE_N_FORMAT()
@@ -801,6 +799,7 @@ static void print_collectd_exec_output(void)
     COLLECTD_STATS_GAUGE_SUB(flow_category_antimalware_count);
     COLLECTD_STATS_GAUGE_SUB(flow_category_crypto_currency_count);
     COLLECTD_STATS_GAUGE_SUB(flow_category_gambling_count);
+    COLLECTD_STATS_GAUGE_SUB(flow_category_health_count);
     COLLECTD_STATS_GAUGE_SUB(flow_category_unknown_count);
 
     COLLECTD_STATS_GAUGE_SUB(flow_confidence_by_port);

diff --git a/examples/c-influxd/c-influxd.c b/examples/c-influxd/c-influxd.c
@@ -170,6 +170,7 @@ static struct
         uint64_t flow_category_antimalware_count;
         uint64_t flow_category_crypto_currency_count;
         uint64_t flow_category_gambling_count;
+        uint64_t flow_category_health_count;
         uint64_t flow_category_unknown_count;
 
         uint64_t flow_confidence_by_port;
@@ -220,15 +221,10 @@ struct global_map
     };
 };
 
-#define INFLUXD_STATS_COUNTER_PTR(member)                                                                              \
-    {                                                                                                                  \
-        .global_stat_inc = &(influxd_statistics.counters.member), NULL                                                 \
-    }
+#define INFLUXD_STATS_COUNTER_PTR(member) {.global_stat_inc = &(influxd_statistics.counters.member), NULL}
 #define INFLUXD_STATS_GAUGE_PTR(member)                                                                                \
-    {                                                                                                                  \
-        .global_stat_inc = &(influxd_statistics.gauges[0].member),                                                     \
-        .global_stat_dec = &(influxd_statistics.gauges[1].member)                                                      \
-    }
+    {.global_stat_inc = &(influxd_statistics.gauges[0].member),                                                        \
+     .global_stat_dec = &(influxd_statistics.gauges[1].member)}
 #define INFLUXD_STATS_COUNTER_INC(member) (influxd_statistics.counters.member++)
 #define INFLUXD_STATS_GAUGE_RES(member) (influxd_statistics.gauges[0].member--)
 #define INFLUXD_STATS_GAUGE_INC(member) (influxd_statistics.gauges[0].member++)
@@ -328,6 +324,7 @@ static struct global_map const categories_map[] = {
     {"Antimalware", INFLUXD_STATS_GAUGE_PTR(flow_category_antimalware_count)},
     {"Crypto_Currency", INFLUXD_STATS_GAUGE_PTR(flow_category_crypto_currency_count)},
     {"Gambling", INFLUXD_STATS_GAUGE_PTR(flow_category_gambling_count)},
+    {"Health", INFLUXD_STATS_GAUGE_PTR(flow_category_health_count)},
     {NULL, INFLUXD_STATS_GAUGE_PTR(flow_category_unknown_count)}};
 
 static struct global_map const confidence_map[] = {
@@ -486,7 +483,7 @@ static int serialize_influx_line(char * buf, size_t siz)
                                              INFLUXDB_FORMAT() INFLUXDB_FORMAT() INFLUXDB_FORMAT() INFLUXDB_FORMAT()
                                                  INFLUXDB_FORMAT() INFLUXDB_FORMAT() INFLUXDB_FORMAT() INFLUXDB_FORMAT()
                                                      INFLUXDB_FORMAT() INFLUXDB_FORMAT() INFLUXDB_FORMAT()
-                                                         INFLUXDB_FORMAT() INFLUXDB_FORMAT_END(),
+                                                         INFLUXDB_FORMAT() INFLUXDB_FORMAT() INFLUXDB_FORMAT_END(),
 
                      "category",
                      INFLUXDB_VALUE_GAUGE(flow_category_unspecified_count),
@@ -528,6 +525,7 @@ static int serialize_influx_line(char * buf, size_t siz)
                      INFLUXDB_VALUE_GAUGE(flow_category_antimalware_count),
                      INFLUXDB_VALUE_GAUGE(flow_category_crypto_currency_count),
                      INFLUXDB_VALUE_GAUGE(flow_category_gambling_count),
+                     INFLUXDB_VALUE_GAUGE(flow_category_health_count),
                      INFLUXDB_VALUE_GAUGE(flow_category_unknown_count));
     CHECK_SNPRINTF_RET(bytes);
 
@@ -656,6 +654,7 @@ static int serialize_influx_line(char * buf, size_t siz)
     INFLUXD_STATS_GAUGE_SUB(flow_category_antimalware_count);
     INFLUXD_STATS_GAUGE_SUB(flow_category_crypto_currency_count);
     INFLUXD_STATS_GAUGE_SUB(flow_category_gambling_count);
+    INFLUXD_STATS_GAUGE_SUB(flow_category_health_count);
     INFLUXD_STATS_GAUGE_SUB(flow_category_unknown_count);
 
     INFLUXD_STATS_GAUGE_SUB(flow_confidence_by_port);

diff --git a/libnDPI b/libnDPI
diff --git a/nDPId.c b/nDPId.c
@@ -485,7 +485,7 @@ static struct
     struct cmdarg custom_risk_domain_file;
     struct cmdarg custom_protocols_file;
     struct cmdarg custom_categories_file;
-    struct cmdarg custom_ja3_file;
+    struct cmdarg custom_ja4_file;
     struct cmdarg custom_sha1_file;
     struct cmdarg collector_address;
     struct cmdarg instance_alias;
@@ -534,7 +534,7 @@ static struct
                    .custom_risk_domain_file = CMDARG_STR(NULL),
                    .custom_protocols_file = CMDARG_STR(NULL),
                    .custom_categories_file = CMDARG_STR(NULL),
-                   .custom_ja3_file = CMDARG_STR(NULL),
+                   .custom_ja4_file = CMDARG_STR(NULL),
                    .custom_sha1_file = CMDARG_STR(NULL),
                    .collector_address = CMDARG_STR(COLLECTOR_UNIX_SOCKET),
                    .instance_alias = CMDARG_STR(NULL),
@@ -591,7 +591,7 @@ struct confopt general_config_map[] = {CONFOPT("netif", &nDPId_options.pcap_file
                                        CONFOPT("riskdomains", &nDPId_options.custom_risk_domain_file),
                                        CONFOPT("protocols", &nDPId_options.custom_protocols_file),
                                        CONFOPT("categories", &nDPId_options.custom_categories_file),
-                                       CONFOPT("ja3", &nDPId_options.custom_ja3_file),
+                                       CONFOPT("ja4", &nDPId_options.custom_ja4_file),
                                        CONFOPT("sha1", &nDPId_options.custom_sha1_file),
                                        CONFOPT("collector", &nDPId_options.collector_address),
                                        CONFOPT("alias", &nDPId_options.instance_alias),
@@ -1543,9 +1543,9 @@ static struct nDPId_workflow * init_workflow(char const * const file_or_device)
     {
         ndpi_load_categories_file(workflow->ndpi_struct, GET_CMDARG_STR(nDPId_options.custom_categories_file), NULL);
     }
-    if (IS_CMDARG_SET(nDPId_options.custom_ja3_file) != 0)
+    if (IS_CMDARG_SET(nDPId_options.custom_ja4_file) != 0)
     {
-        ndpi_load_malicious_ja3_file(workflow->ndpi_struct, GET_CMDARG_STR(nDPId_options.custom_ja3_file));
+        ndpi_load_malicious_ja4_file(workflow->ndpi_struct, GET_CMDARG_STR(nDPId_options.custom_ja4_file));
     }
     if (IS_CMDARG_SET(nDPId_options.custom_sha1_file) != 0)
     {
@@ -5320,8 +5320,7 @@ static void print_usage(char const * const arg0)
         "\t  \tDefault: disabled\n"
         "\t-C\tLoad a nDPI custom categories file.\n"
         "\t  \tDefault: disabled\n"
-        "\t-J\tLoad a nDPI JA3 hash blacklist file.\n"
-        "\t  \tSee: https://sslbl.abuse.ch/blacklist/ja3_fingerprints.csv\n"
+        "\t-J\tLoad a nDPI JA4 hash blacklist file.\n"
         "\t  \tDefault: disabled\n"
         "\t-S\tLoad a nDPI SSL SHA1 hash blacklist file.\n"
         "\t  \tSee: https://sslbl.abuse.ch/blacklist/sslblacklist.csv\n"
@@ -5490,7 +5489,7 @@ static int nDPId_parse_options(int argc, char ** argv)
                 set_cmdarg_string(&nDPId_options.custom_categories_file, optarg);
                 break;
             case 'J':
-                set_cmdarg_string(&nDPId_options.custom_ja3_file, optarg);
+                set_cmdarg_string(&nDPId_options.custom_ja4_file, optarg);
                 break;
             case 'S':
                 set_cmdarg_string(&nDPId_options.custom_sha1_file, optarg);

diff --git a/ndpid.conf.example b/ndpid.conf.example
@@ -13,7 +13,7 @@
 #riskdomains = /path/to/libnDPI/example/risky_domains.txt
 #protocols   = /path/to/libnDPI/example/protos.txt
 #categories  = /path/to/libnDPI/example/categories.txt
-#ja3         = /path/to/libnDPI/example/ja3_fingerprints.csv
+#ja4         = /path/to/libnDPI/example/ja4_fingerprints.csv
 #sha1        = /path/to/libnDPI/example/sha1_fingerprints.csv
 
 # Collector endpoint as UNIX socket (usually nDPIsrvd)

diff --git a/packages/openwrt/net/nDPId-testing/nDPId-testing.config b/packages/openwrt/net/nDPId-testing/nDPId-testing.config
@@ -33,7 +33,7 @@ config nDPId
 	#option udp_connect '127.0.0.1:31337'
 	#option proto_file ''
 	#option cat_file ''
-	#option ja3_file ''
+	#option ja4_file ''
 	#option ssl_file ''
 	#option alias ''
 	#option analysis 0

diff --git a/packages/openwrt/net/nDPId-testing/nDPId-testing.init b/packages/openwrt/net/nDPId-testing/nDPId-testing.init
@@ -71,7 +71,7 @@ start_ndpid_instance() {
 	args="$args$(print_arg_bool "$cfg" 'use_poll' '-e')"
 	args="$args$(print_arg_str "$cfg" 'proto_file' '-P')"
 	args="$args$(print_arg_str "$cfg" 'cat_file' '-C')"
-	args="$args$(print_arg_str "$cfg" 'ja3_file' '-J')"
+	args="$args$(print_arg_str "$cfg" 'ja4_file' '-J')"
 	args="$args$(print_arg_str "$cfg" 'ssl_file' '-S')"
 	args="$args$(print_arg_str "$cfg" 'alias' '-a')"
 	args="$args$(print_arg_bool "$cfg" 'analysis' '-A')"
@@ -118,7 +118,7 @@ validate_ndpid_section() {
 		'udp_connect:string' \
 		'proto_file:string' \
 		'cat_file:string' \
-		'ja3_file:string' \
+		'ja4_file:string' \
 		'ssl_file:string' \
 		'alias:string' \
 		'analysis:bool:0' \

diff --git a/schema/flow_event_schema.json b/schema/flow_event_schema.json
@@ -244,6 +244,9 @@
                 "proto_by_ip_id": {
                     "type": "number"
                 },
+                "stream_content": {
+                    "type": "string"
+                },
                 "category": {
                     "type": "string",
                     "enum": [
@@ -254,7 +257,7 @@
                         "Shopping", "Productivity", "FileSharing", "ConnCheck", "IoT-Scada",
                         "VirtAssistant", "Cybersecurity", "AdultContent", "Mining", "Malware",
                         "Advertisement", "Banned_Site", "Site_Unavailable", "Allowed_Site",
-                        "Antimalware", "Crypto_Currency", "Gambling"
+                        "Antimalware", "Crypto_Currency", "Gambling", "Health"
                     ]
                 },
                 "category_id": {
@@ -1316,7 +1319,7 @@
                                     "properties": {
                                         "total": { "type": "number", "minimum": 10, "maximum": 610 },
                                         "client": { "type": "number", "minimum": 5, "maximum": 485 },
-                                        "server": { "type": "number", "minimum": 5, "maximum": 130 },
+                                        "server": { "type": "number", "minimum": 5, "maximum": 135 },
                                         "additionalProperties": false
                                     }
                                 }
@@ -1411,6 +1414,9 @@
                 "mdns": {
                     "type": "object"
                 },
+                "mikrotik": {
+                    "type": "object"
+                },
                 "natpmp": {
                     "type": "object"
                 },
@@ -1450,6 +1456,18 @@
                 "ftp": {
                     "type": "object"
                 },
+                "sip": {
+                    "type": "object",
+                    "properties": {
+                        "from": {
+                            "type": "string"
+                        },
+                        "to": {
+                            "type": "string"
+                        }
+                    },
+                    "additionalProperties": false
+                },
                 "snmp": {
                     "type": "object"
                 },

diff --git a/test/configs/disable_metadata.ndpiconf b/test/configs/disable_metadata.ndpiconf
@@ -1,5 +1,4 @@
 [protos]
 tls.metadata.sha1_fingerprint=0
-tls.metadata.ja3c_fingerprint=0
 tls.metadata.ja3s_fingerprint=0
 tls.metadata.ja4c_fingerprint=0