Normalization of host_server_name (ntop#2299)

* Normalization of host_server_name

The ndpi_hostname_sni_set() function replaces all non-printable
characters with the "?" character and removing whitespace characters
at the end of the line.

* Added conditional hostname normalization.

src/include/ndpi_private.h

@@ -430,7 +430,7 @@ void change_category(struct ndpi_detection_module_struct *ndpi_struct,
 		     ndpi_protocol_category_t protocol_category);
 
 
-char *ndpi_hostname_sni_set(struct ndpi_flow_struct *flow, const u_int8_t *value, size_t value_len);
+char *ndpi_hostname_sni_set(struct ndpi_flow_struct *flow, const u_int8_t *value, size_t value_len, int normalize);
 char *ndpi_user_agent_set(struct ndpi_flow_struct *flow, const u_int8_t *value, size_t value_len);
 
 void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struct,

src/include/ndpi_typedefs.h

@@ -263,6 +263,11 @@ typedef u_int32_t ndpi_ndpi_mask;
 #define NDPI_NUM_FDS_BITS     16
 #endif
 
+#define NDPI_HOSTNAME_NORM_LC 1
+#define NDPI_HOSTNAME_NORM_REPLACE_IC 2
+#define NDPI_HOSTNAME_NORM_STRIP_EOLSP 4
+#define NDPI_HOSTNAME_NORM_ALL (NDPI_HOSTNAME_NORM_LC | NDPI_HOSTNAME_NORM_REPLACE_IC | NDPI_HOSTNAME_NORM_STRIP_EOLSP)
+
 typedef struct ndpi_protocol_bitmask_struct {
   ndpi_ndpi_mask fds_bits[NDPI_NUM_FDS_BITS];
 } ndpi_protocol_bitmask_struct_t;

src/lib/ndpi_main.c

@@ -10580,16 +10580,38 @@ ndpi_risk_info* ndpi_risk2severity(ndpi_risk_enum risk) {
 /* ******************************************************************** */
 
 char *ndpi_hostname_sni_set(struct ndpi_flow_struct *flow,
-			    const u_int8_t *value, size_t value_len) {
+			    const u_int8_t *value, size_t value_len,
+			    int normalize) {
   char *dst;
   size_t len, i;
 
   len = ndpi_min(value_len, sizeof(flow->host_server_name) - 1);
   dst = flow->host_server_name;
 
-  for(i = 0; i < len; i++)
-    dst[i] = tolower(value[value_len - len + i]);
-  dst[i] = '\0';
+  if(!normalize) {
+    memcpy(dst,&value[value_len - len],len);
+    dst[len] = '\0';
+  } else {
+    for(i = 0; i < len; i++) {
+      char c = value[value_len - len + i];
+      if(!c) break;
+      if(normalize & NDPI_HOSTNAME_NORM_LC) c = tolower(c);
+      if(normalize & NDPI_HOSTNAME_NORM_REPLACE_IC) {
+        if (c == '\t') c = ' ';
+        if (ndpi_isprint(c) == 0)
+            c = '?';
+      }
+      dst[i] = c;
+    }
+
+    dst[i] = '\0';
+    if(normalize & NDPI_HOSTNAME_NORM_STRIP_EOLSP) {
+      /* Removing spaces at the end of a line */
+      while(i > 0 && dst[i-1] == ' ')
+        dst[--i] = '\0';
+    }
+  }
+
 
   return dst;
 }

src/lib/protocols/collectd.c

@@ -105,7 +105,7 @@ static int ndpi_int_collectd_dissect_hostname(struct ndpi_flow_struct * const fl
                                               struct ndpi_packet_struct const * const packet,
                                               u_int16_t block_offset, u_int16_t block_length)
 {
-  return (ndpi_hostname_sni_set(flow, &packet->payload[4], block_length) == NULL);
+  return (ndpi_hostname_sni_set(flow, &packet->payload[4], block_length, NDPI_HOSTNAME_NORM_ALL) == NULL);
 }
 
 static int ndpi_int_collectd_dissect_username(struct ndpi_flow_struct * const flow,

src/lib/protocols/dhcp.c

@@ -180,7 +180,7 @@ static void ndpi_search_dhcp_udp(struct ndpi_detection_module_struct *ndpi_struc
             NDPI_LOG_DBG2(ndpi_struct, "[DHCP] '%.*s'\n",name,len);
 	      //	    while(j < len) { printf( "%c", name[j]); j++; }; printf("\n");
 #endif
-            ndpi_hostname_sni_set(flow, name, len);
+            ndpi_hostname_sni_set(flow, name, len, NDPI_HOSTNAME_NORM_ALL);
 	  }
 
           i += len + 2;

src/lib/protocols/dns.c

@@ -764,7 +764,7 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
 
     u_int8_t hostname_is_valid = ndpi_grab_dns_name(packet, &off, _hostname, sizeof(_hostname), &len, is_mdns);
 
-    ndpi_hostname_sni_set(flow, (const u_int8_t *)_hostname, len);
+    ndpi_hostname_sni_set(flow, (const u_int8_t *)_hostname, len, is_mdns ? NDPI_HOSTNAME_NORM_LC : NDPI_HOSTNAME_NORM_ALL);
 
     if (hostname_is_valid == 0)
       ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS, NULL);

src/lib/protocols/fastcgi.c

@@ -138,7 +138,7 @@ static int fcgi_parse_params(struct ndpi_flow_struct * const flow,
 
   flow->http.method = ndpi_http_str2method((const char*)packet->http_method.ptr,
                                            (u_int16_t)packet->http_method.len);
-  ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len);
+  ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len, NDPI_HOSTNAME_NORM_ALL);
   ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
 
   if (flow->http.url == NULL && packet->http_url_name.len > 0)

src/lib/protocols/http.c

@@ -950,7 +950,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
 		  packet->host_line.len, packet->host_line.ptr);
 
     /* Copy result for nDPI apps */
-    ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len);
+    ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len, NDPI_HOSTNAME_NORM_ALL);
 
     if(strlen(flow->host_server_name) > 0) {
       char *double_col;

src/lib/protocols/mail_smtp.c

@@ -151,7 +151,7 @@ static void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_
 		   && (packet->line[a].ptr[i+1] != '\n')) {
 		  len = i-4;
 		  /* Copy result for nDPI apps */
-		  ndpi_hostname_sni_set(flow, &packet->line[a].ptr[4], len);
+		  ndpi_hostname_sni_set(flow, &packet->line[a].ptr[4], len, NDPI_HOSTNAME_NORM_ALL);
 		  NDPI_LOG_DBG(ndpi_struct, "SMTP: hostname [%s]\n", flow->host_server_name);
 
 		  if (ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP,

src/lib/protocols/mgcp.c

@@ -88,10 +88,10 @@ static void ndpi_search_mgcp(struct ndpi_detection_module_struct *ndpi_struct, s
       endpoint_hostname = ndpi_strnstr(endpoint, "@", packet->payload_packet_len - ((u_int8_t const *)endpoint - packet->payload));
       if (endpoint_hostname == NULL || endpoint_hostname >= mgcp)
       {
-        ndpi_hostname_sni_set(flow, (u_int8_t const *)endpoint, (mgcp - endpoint) - 1);
+        ndpi_hostname_sni_set(flow, (u_int8_t const *)endpoint, (mgcp - endpoint) - 1, NDPI_HOSTNAME_NORM_ALL);
       } else {
         endpoint_hostname++;
-        ndpi_hostname_sni_set(flow, (u_int8_t const *)endpoint_hostname, (mgcp - endpoint_hostname) - 1);
+        ndpi_hostname_sni_set(flow, (u_int8_t const *)endpoint_hostname, (mgcp - endpoint_hostname) - 1, NDPI_HOSTNAME_NORM_ALL);
       }
       return;
     }

src/lib/protocols/munin.c

@@ -71,7 +71,7 @@ static void ndpi_search_munin(struct ndpi_detection_module_struct *ndpi_struct,
   size_t host_len = packet->payload_packet_len - NDPI_STATICSTRING_LEN(munin_prefix) - 1;
   if (host_len > 0)
   {
-    ndpi_hostname_sni_set(flow, packet->payload + NDPI_STATICSTRING_LEN(munin_prefix), host_len);
+    ndpi_hostname_sni_set(flow, packet->payload + NDPI_STATICSTRING_LEN(munin_prefix), host_len, NDPI_HOSTNAME_NORM_ALL);
   } else {
     ndpi_set_risk(ndpi_struct, flow, NDPI_MALFORMED_PACKET, "Missing Munin Hostname");
   }

src/lib/protocols/netbios.c

@@ -103,7 +103,7 @@ static void ndpi_int_netbios_add_connection(struct ndpi_detection_module_struct
   if((off < packet->payload_packet_len)
      && ndpi_netbios_name_interpret((unsigned char*)&packet->payload[off],
 		 (u_int)(packet->payload_packet_len - off), name, sizeof(name)-1) > 0) {
-      ndpi_hostname_sni_set(flow, (const u_int8_t *)name, strlen((char *)name));
+      ndpi_hostname_sni_set(flow, (const u_int8_t *)name, strlen((char *)name), NDPI_HOSTNAME_NORM_ALL);
 
       ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1, 1);
   }

src/lib/protocols/quic.c

@@ -1451,7 +1451,7 @@ void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
 #endif
     if(memcmp(tag, "SNI\0", 4) == 0) {
 
-      ndpi_hostname_sni_set(flow, &crypto_data[tag_offset_start + prev_offset], len);
+      ndpi_hostname_sni_set(flow, &crypto_data[tag_offset_start + prev_offset], len, NDPI_HOSTNAME_NORM_ALL);
 
       NDPI_LOG_DBG2(ndpi_struct, "SNI: [%s]\n",
                     flow->host_server_name);

src/lib/protocols/sd_rtn.c

@@ -51,7 +51,7 @@ static int ndpi_int_sd_rtn_dissect_sni(struct ndpi_flow_struct * const flow,
     return -1;
   }
 
-  ndpi_hostname_sni_set(flow, &payload[19], sni_len);
+  ndpi_hostname_sni_set(flow, &payload[19], sni_len, NDPI_HOSTNAME_NORM_ALL);
 
   return 0;
 }

src/lib/protocols/ssdp.c

@@ -47,7 +47,7 @@ static void ssdp_parse_lines(struct ndpi_detection_module_struct
 
   /* Save host which provides a service if available */
   if (packet->host_line.ptr != NULL && packet->host_line.len > 0) {
-    if (ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len) == NULL)
+    if (ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len, NDPI_HOSTNAME_NORM_ALL) == NULL)
     {
       NDPI_LOG_DBG2(ndpi_struct, "Could not set SSDP host\n");
     }

src/lib/protocols/stun.c

@@ -270,7 +270,7 @@ int is_stun(struct ndpi_detection_module_struct *ndpi_struct,
 
     case 0x0014: /* Realm */
       if(flow->host_server_name[0] == '\0') {
-        ndpi_hostname_sni_set(flow, payload + off + 4, ndpi_min(len, payload_length - off - 4));
+        ndpi_hostname_sni_set(flow, payload + off + 4, ndpi_min(len, payload_length - off - 4), NDPI_HOSTNAME_NORM_ALL);
         NDPI_LOG_DBG(ndpi_struct, "Realm [%s]\n", flow->host_server_name);
 
         if(strstr(flow->host_server_name, "google.com") != NULL) {

src/lib/protocols/tls.c

@@ -2253,7 +2253,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
 		  len = (packet->payload[offset+extension_offset+3] << 8) + packet->payload[offset+extension_offset+4];
 
 		  if((offset+extension_offset+5+len) <= packet->payload_packet_len) {
-		    char *sni = ndpi_hostname_sni_set(flow, &packet->payload[offset+extension_offset+5], len);
+		    char *sni = ndpi_hostname_sni_set(flow, &packet->payload[offset+extension_offset+5], len, NDPI_HOSTNAME_NORM_ALL);
 		    int sni_len = strlen(sni);
 #ifdef DEBUG_TLS
 		    printf("[TLS] SNI: [%s]\n", sni);

src/lib/protocols/whoisdas.c

@@ -42,7 +42,7 @@ static void ndpi_search_whois_das(struct ndpi_detection_module_struct *ndpi_stru
       ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WHOIS_DAS, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
 
       if((dport == 43) || (dport == 4343)) { /* Request */
-        ndpi_hostname_sni_set(flow, &packet->payload[0], packet->payload_packet_len - 2); /* Skip \r\n */
+        ndpi_hostname_sni_set(flow, &packet->payload[0], packet->payload_packet_len - 2, NDPI_HOSTNAME_NORM_ALL); /* Skip \r\n */
         NDPI_LOG_INFO(ndpi_struct, "[WHOIS/DAS] %s\n", flow->host_server_name);
       }
       return;

src/lib/protocols/xiaomi.c

@@ -63,9 +63,9 @@ static void xiaomi_dissect_metadata(struct ndpi_detection_module_struct *ndpi_st
         /* If "domain:port", strip the port */
         ptr = ndpi_strnstr((const char *)&payload[offset], ":", len);
         if(ptr == NULL)
-          ndpi_hostname_sni_set(flow, &payload[offset], len);
+          ndpi_hostname_sni_set(flow, &payload[offset], len, NDPI_HOSTNAME_NORM_ALL);
         else
-          ndpi_hostname_sni_set(flow, &payload[offset], (const u_int8_t *)ptr - &payload[offset]);
+          ndpi_hostname_sni_set(flow, &payload[offset], (const u_int8_t *)ptr - &payload[offset], NDPI_HOSTNAME_NORM_ALL);
         break;
 
       case 0x32: /* Radio access technology (+ APN) */