Add an heuristic to detect/ignore some anomalous TCP ACK packets

In some networks, there are some anomalous TCP flows where the smallest
ACK packets have some kind of zero padding.
It looks like the IP and TCP headers in those frames wrongly consider the
0x00 Ethernet padding bytes as part of the TCP payload.
While this kind of packets is perfectly valid per-se, in some conditions
they might be treated by the TCP reassembler logic as (partial) overlaps,
deceiving the classification engine.
Add an heuristic to detect these packets and to ignore them, allowing
correct detection/classification.

This heuristic is configurable. Default value:
* in the library, it is disabled
* in `ndpiReader` and in the fuzzers, it is enabled (to ease testing)

Credit to @vel21ripn for the initial patch.

Close #1946

example/ndpiReader.c

@@ -98,7 +98,7 @@ char *_debug_protocols = NULL;
 char *_disabled_protocols = NULL;
 int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS];
 static u_int8_t stats_flag = 0;
-ndpi_init_prefs init_prefs = ndpi_no_prefs;
+ndpi_init_prefs init_prefs = ndpi_no_prefs | ndpi_enable_tcp_ack_payload_heuristic;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 24 /* 8 is enough for most protocols, Signal and SnapchatCall require more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
 static u_int32_t pcap_analysis_duration = (u_int32_t)-1;

fuzz/fuzz_ndpi_reader.c

@@ -18,7 +18,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0;
 u_int8_t enable_flow_stats = 1;
 u_int8_t human_readeable_string_len = 5;
 u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
-ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus;
+ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic;
 int enable_malloc_bins = 1;
 int malloc_size_stats = 0;
 int max_malloc_bins = 0;

src/include/ndpi_typedefs.h

@@ -1273,6 +1273,8 @@ struct ndpi_detection_module_struct {
 
   u_int32_t aggressiveness_ookla;
 
+  int tcp_ack_paylod_heuristic;
+
   u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */
   ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
 
@@ -1691,6 +1693,17 @@ typedef enum {
     ndpi_dont_init_risk_ptree      = (1 << 14),
     ndpi_dont_load_cachefly_list   = (1 << 15),
     ndpi_track_flow_payload        = (1 << 16),
+    /* In some networks, there are some anomalous TCP flows where
+       the smallest ACK packets have some kind of zero padding.
+       It looks like the IP and TCP headers in those frames wrongly consider the
+       0x00 Ethernet padding bytes as part of the TCP payload.
+       While this kind of packets is perfectly valid per-se, in some conditions
+       they might be treated by the TCP reassembler logic as (partial) overlaps,
+       deceiving the classification engine.
+       Add an heuristic to detect these packets and to ignore them, allowing
+       correct detection/classification.
+       See #1946 for other details */
+    ndpi_enable_tcp_ack_payload_heuristic = (1 << 17),
   } ndpi_prefs;
 
 typedef struct {

src/lib/ndpi_main.c

@@ -2951,6 +2951,9 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
 
   ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS;
 
+  if(prefs & ndpi_enable_tcp_ack_payload_heuristic)
+    ndpi_str->tcp_ack_paylod_heuristic = 1;
+
   for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++)
     ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u",
 	     (unsigned int) (i + 1));
@@ -5492,6 +5495,20 @@ static u_int8_t ndpi_is_multi_or_broadcast(struct ndpi_packet_struct *packet) {
 
 /* ************************************************ */
 
+static int tcp_ack_padding(struct ndpi_packet_struct *packet) {
+  const struct ndpi_tcphdr *tcph = packet->tcp;
+  if(tcph && tcph->ack && !tcph->psh &&
+     packet->payload_packet_len < 8 &&
+     packet->payload_packet_len > 1 /* To avoid TCP keep-alives */) {
+    int i;
+    for(i = 0; i < packet->payload_packet_len; i++)
+      if(packet->payload[i] != 0)
+        return 0;
+    return 1;
+  }
+  return 0;
+}
+
 void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
 			      struct ndpi_flow_struct *flow) {
   if(!flow) {
@@ -5580,7 +5597,10 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
 	}
       }
 
-      if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 ||
+      if(ndpi_str->tcp_ack_paylod_heuristic && tcp_ack_padding(packet)) {
+        NDPI_LOG_DBG2(ndpi_str, "TCP ACK with zero padding. Ignoring\n");
+        packet->tcp_retransmission = 1;
+      } else if(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0 ||
 	 (tcph->syn && flow->packet_counter == 0)) {
 	/* initialize tcp sequence counters */
 	/* the ack flag needs to be set to get valid sequence numbers from the other

tests/cfgs/default/result/heuristic_tcp_ack_payload.pcap.out

@@ -0,0 +1,36 @@
+Guessed flow protos:	0
+
+DPI Packets (TCP):	41	(10.25 pkts/flow)
+Confidence DPI              : 4 (flows)
+Num dissector calls: 15 (3.75 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/0/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/6/0 (insert/search/found)
+LRU cache mining:     0/0/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          8/1 (search/found)
+Automa domain:        8/0 (search/found)
+Automa tls cert:      2/0 (search/found)
+Automa risk mask:     0/0 (search/found)
+Automa common alpns:  6/6 (search/found)
+Patricia risk mask:   8/0 (search/found)
+Patricia risk:        8/0 (search/found)
+Patricia protocols:   6/2 (search/found)
+
+TLS	160	63162	3
+WindowsUpdate	19	2638	1
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+	1	 194.226.199.103          	 1      
+	2	 194.226.199.21           	 1      
+	3	 194.226.199.61           	 1      
+
+
+	1	TCP 194.226.199.61:27453 <-> 35.241.9.150:443 [proto: 91/TLS][IP: 284/GoogleCloud][Encrypted][Confidence: DPI][DPI packets: 8][cat: Web/5][36 pkts/3477 bytes <-> 42 pkts/37330 bytes][Goodput ratio: 44/94][171.42 sec][Hostname/SNI: firefox.settings.services.mozilla.com][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][bytes ratio: -0.830 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4196/3653 58250/58245 14929/14067][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 97/889 375/2878 73/1070][TLSv1.2][JA3C: fb0aa01abe9d8e4037eb3473ca6e2dca][ServerNames: firefox.settings.services.mozilla.com,main-2-cdn.prod.kinto.prod.cloudops.mozgcp.net][JA3S: 9d9ce860f1b1cbef07b019450cb368d8][Issuer: C=US, O=Let's Encrypt, CN=R3][Subject: CN=main-2-cdn.prod.kinto.prod.cloudops.mozgcp.net][Certificate SHA-1: 30:0D:22:77:6E:DA:4E:99:3E:AF:8A:D0:5C:7D:97:51:8B:E6:22:11][Firefox][Validity: 2023-04-04 08:33:24 - 2023-07-03 08:33:23][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 49,16,2,0,3,3,1,0,1,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,11]
+	2	TCP 194.226.199.21:58155 <-> 52.18.127.189:443 [proto: 91/TLS][IP: 265/AmazonAWS][Encrypted][Confidence: DPI][DPI packets: 13][cat: Web/5][28 pkts/6789 bytes <-> 35 pkts/8995 bytes][Goodput ratio: 78/79][130.64 sec][Hostname/SNI: bitrix.info][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: h2][TLS Supported Versions: GREASE;TLSv1.3;TLSv1.2][bytes ratio: -0.140 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 5498/4834 45102/45058 12717/11564][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 242/257 1547/2974 352/535][TLSv1.2][JA3C: ab205d804ecf934209c2a1bb94f817e0][JA3S: bfc90d56141386ee83b56cda231cccfc][Chrome][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 46,22,6,6,0,0,0,0,0,0,1,0,6,0,1,1,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,1]
+	3	TCP 194.226.199.103:62580 <-> 217.69.139.59:443 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 8][cat: Web/5][11 pkts/1346 bytes <-> 8 pkts/5225 bytes][Goodput ratio: 55/92][7.28 sec][Hostname/SNI: portal.mail.ru][(Advertised) ALPNs: h2;http/1.1][(Negotiated) ALPN: http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.590 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 808/58 5455/213 1711/90][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 122/653 623/2897 162/957][TLSv1.2][JA3C: e669667efb41c36f714c309243f41ca7][ServerNames: *.mail.ru,mail.ru][JA3S: 2b33c1374db4ddf06942f92373c0b54b][Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign RSA OV SSL CA 2018][Subject: C=RU, ST=Moscow, L=Moscow, O=VK LLC, CN=*.mail.ru][Certificate SHA-1: 9F:A2:43:EA:AA:62:15:13:44:0D:15:75:17:47:4C:6B:E5:8E:10:1E][Firefox][Validity: 2022-10-20 09:52:31 - 2023-11-21 09:52:30][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 68,0,0,5,0,0,0,0,11,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,5]
+	4	TCP 194.226.199.226:34101 <-> 8.247.226.126:80 [proto: 7.147/HTTP.WindowsUpdate][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 12][cat: SoftwareUpdate/19][7 pkts/896 bytes <-> 12 pkts/1742 bytes][Goodput ratio: 56/62][0.04 sec][Hostname/SNI: 3.tlu.dl.delivery.mp.microsoft.com][bytes ratio: -0.321 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/1 12/11 6/3][Pkt Len c2s/s2c min/avg/max/stddev: 56/60 128/145 550/1076 172/281][URL: 3.tlu.dl.delivery.mp.microsoft.com/filestreamingservice/files/b4f27514-1618-47a0-bcd4-5fcb469edb63?P1=1681888058&P2=404&P3=2&P4=VJ2Qv%2bUXzBGOULZmyshxlc8XXx4pLl7hoFcLgf1iS33rDGfm0tCVrTPvZN8tn8yWBSrA0idwdtOBFLQMjZCUkw%3d%3d][StatusCode: 0][User-Agent: Microsoft-Delivery-Optimization/10.0][PLAIN TEXT (GET /filestreamingservice/files)][Plen Bins: 89,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]