Improve handling of custom rules (#2276)

Avoid collisions between user-ids and internal-ids protocols in the `example/protos.txt` file. Add a new value for the classification confidence: `NDPI_CONFIDENCE_CUSTOM_RULE` With `./example/ndpiReader -p example/protos.txt -H` we now see also the custom protocols and their internal/external ids: ``` nDPI supported protocols: Id Userd-id Protocol Layer_4 Nw_Proto Breed Category 0 0 Unknown TCP X Unrated Unspecified ... 387 387 Mumble UDP X Fun VoIP 388 388 iSCSI TCP Acceptable Unspecified 389 389 Kibana TCP Acceptable Unspecified 390 390 TestProto TCP Acceptable Unspecified 391 391 HomeRouter TCP Acceptable Unspecified 392 392 CustomProtocol TCP Acceptable Unspecified 393 393 AmazonPrime TCP Acceptable Unspecified 394 394 CustomProtocolA TCP Acceptable Unspecified 395 395 CustomProtocolB TCP Acceptable Unspecified 396 800 CustomProtocolC TCP Acceptable Unspecified 397 1024 CustomProtocolD TCP Acceptable Unspecified 398 2048 CustomProtocolE TCP Acceptable Unspecified 399 2049 CustomProtocolF TCP Acceptable Unspecified 400 2050 CustomProtocolG TCP Acceptable Unspecified 401 65535 CustomProtocolH TCP Acceptable Unspecified ``` We likely need to take a better look in general at the iteration between internal and external protocols ids... This PR fixes the issue observed in #2274 (comment) and in #2275.
ntop · Jan 21, 2024 · 82e8bf9 · 82e8bf9
1 parent 5620e10
commit 82e8bf9
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 34 deletions.
diff --git a/example/ndpiReader.c b/example/ndpiReader.c
@@ -609,6 +609,10 @@ static void help(u_int long_help) {
   struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module();
   NDPI_BITMASK_SET_ALL(all);
   ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all);
+
+  if(_protoFilePath != NULL)
+    ndpi_load_protocols_file(ndpi_info_mod, _protoFilePath);
+
   ndpi_finalize_initialization(ndpi_info_mod);
 
   printf("\nProtocols configuration parameters:\n");
@@ -635,8 +639,8 @@ static void help(u_int long_help) {
            sizeof(((struct ndpi_flow_struct *)0)->protos));
 
     printf("\n\nnDPI supported protocols:\n");
-    printf("%3s %-22s %-10s %-8s %-12s %s\n",
-	   "Id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category");
+    printf("%3s %8s %-22s %-10s %-8s %-12s %s\n",
+	   "Id", "Userd-id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category");
     num_threads = 1;
 
     ndpi_dump_protocols(ndpi_info_mod, stdout);
@@ -4126,7 +4130,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
 
   if(!quiet_mode) printf("\n\nDetected protocols:\n");
   for(i = 0; i <= ndpi_get_num_supported_protocols(ndpi_thread_info[0].workflow->ndpi_struct); i++) {
-    ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct, i);
+    ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct,
+                                                       ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i));
 
     if(cumulative_stats.protocol_counter[i] > 0) {
       breed_stats_bytes[breed] += (long long unsigned int)cumulative_stats.protocol_counter_bytes[i];
@@ -4135,7 +4140,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
 
       if(results_file)
 	fprintf(results_file, "%s\t%llu\t%llu\t%u\n",
-		ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct, i),
+		ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct,
+				    ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i)),
 		(long long unsigned int)cumulative_stats.protocol_counter[i],
 		(long long unsigned int)cumulative_stats.protocol_counter_bytes[i],
 		cumulative_stats.protocol_flows[i]);

diff --git a/example/protos.txt b/example/protos.txt
@@ -50,15 +50,18 @@ ip:213.75.170.11/32:443@CustomProtocol
 ip:8.248.73.247:443@AmazonPrime
 ip:54.80.47.130@AmazonPrime
 
+#You can specify a protocol Id. In that case you probably want to avoid conflict with internal ids.
+#You can use any number up to 65535
+
 ip:3.3.3.3:443@CustomProtocolA
 ip:3.3.3.3:444@CustomProtocolB
-ip:3.3.3.3:446@CustomProtocolC=400
+ip:3.3.3.3:446@CustomProtocolC=800
 
-ipv6:[3ffe:507:0:1:200:86ff:fe05:80da]@CustomProtocolD
-ipv6:[247f:855b:5e16:3caf::]/64:100@CustomProtocolE
-ipv6:[247f:855b:5e16:3caf::]/64@CustomProtocolF
-ipv6:[fe80::76ac:b9ff:fe6c:c124]:12717@CustomProtocolG
-ipv6:[fe80::76ac:b9ff:fe6c:c124]:12718@CustomProtocolH
+ipv6:[3ffe:507:0:1:200:86ff:fe05:80da]@CustomProtocolD=1024
+ipv6:[247f:855b:5e16:3caf::]/64:100@CustomProtocolE=2048
+ipv6:[247f:855b:5e16:3caf::]/64@CustomProtocolF=2049
+ipv6:[fe80::76ac:b9ff:fe6c:c124]:12717@CustomProtocolG=2050
+ipv6:[fe80::76ac:b9ff:fe6c:c124]:12718@CustomProtocolH=65535
 
 #
 # You can use symbolic IP addreses if you want

diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h
@@ -958,6 +958,7 @@ typedef enum {
   NDPI_CONFIDENCE_DPI,                      /* Deep packet inspection */
   NDPI_CONFIDENCE_MATCH_BY_IP,              /* Classification obtained looking only at the IP addresses */
   NDPI_CONFIDENCE_DPI_AGGRESSIVE,           /* Aggressive DPI: it might be a false positive */
+  NDPI_CONFIDENCE_CUSTOM_RULE,              /* Matching a custom rules */
 
   /*
     IMPORTANT

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
@@ -7939,7 +7939,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
       /* This is a custom protocol and it has priority over everything else */
       ret->master_protocol = NDPI_PROTOCOL_UNKNOWN,
       ret->app_protocol = flow->guessed_protocol_id;
-      flow->confidence = NDPI_CONFIDENCE_MATCH_BY_PORT; /* TODO */
+      flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
       ndpi_fill_protocol_category(ndpi_str, flow, ret);
       return(-1);
     }
@@ -7951,6 +7951,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
         *ret = ndpi_detection_giveup(ndpi_str, flow, &protocol_was_guessed);
       }
 
+      flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
       ndpi_fill_protocol_category(ndpi_str, flow, ret);
       return(-1);
     }
@@ -7964,6 +7965,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
 
     flow->num_dissector_calls += ndpi_check_flow_func(ndpi_str, flow, &ndpi_selection_packet);
 
+    flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
     ndpi_fill_protocol_category(ndpi_str, flow, ret);
     return(-1);
   }
@@ -8288,8 +8290,9 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
 							    packetlen, current_time_ms,
 							    input_info);
 
-  p.master_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.master_protocol),
-    p.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.app_protocol);
+  p.master_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.master_protocol);
+  p.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.app_protocol);
+  p.protocol_by_ip = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.protocol_by_ip);
 
   return(p);
 }
@@ -9173,6 +9176,9 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)
   case NDPI_CONFIDENCE_DPI_AGGRESSIVE:
     return "DPI (aggressive)";
 
+  case NDPI_CONFIDENCE_CUSTOM_RULE:
+    return "Match by custom rule";
+
   default:
     return NULL;
   }
@@ -9353,8 +9359,9 @@ void ndpi_dump_protocols(struct ndpi_detection_module_struct *ndpi_str, FILE *du
   if(!ndpi_str || !dump_out) return;
 
   for(i = 0; i < (int) ndpi_str->ndpi_num_supported_protocols; i++)
-    fprintf(dump_out, "%3d %-22s %-10s %-8s %-12s %s\n",
-	   i, ndpi_str->proto_defaults[i].protoName,
+    fprintf(dump_out, "%3d %8d %-22s %-10s %-8s %-12s %s\n",
+	   i, ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, i),
+	   ndpi_str->proto_defaults[i].protoName,
 	   ndpi_get_l4_proto_name(ndpi_get_l4_proto_info(ndpi_str, i)),
 	   ndpi_str->proto_defaults[i].isAppProtocol ? "" : "X",
 	   ndpi_get_proto_breed_name(ndpi_str, ndpi_str->proto_defaults[i].protoBreed),

diff --git a/tests/cfgs/default/result/custom_rules_ipv6.pcapng.out b/tests/cfgs/default/result/custom_rules_ipv6.pcapng.out
@@ -1,5 +1,5 @@
 DPI Packets (UDP):	5	(1.00 pkts/flow)
-Confidence Unknown          : 5 (flows)
+Confidence Match by custom rule: 5 (flows)
 Num dissector calls: 0 (0.00 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
@@ -21,17 +21,16 @@ Patricia risk IPv6:   0/0 (search/found)
 Patricia protocols:   0/0 (search/found)
 Patricia protocols IPv6: 9/5 (search/found)
 
-Unknown	1	1287	1
 CustomProtocolD	2	600	1
+CustomProtocolE	1	1287	1
 CustomProtocolF	1	1287	1
 CustomProtocolG	1	318	1
 CustomProtocolH	1	318	1
 
-Acceptable                       5 2523          4            
-Unrated                          1 1287          1            
+Acceptable                       6 3810          5            
 
-	1	UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:100 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:1991 [proto: 400/CustomProtocolC][IP: 400/CustomProtocolC][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No client to server traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
-	2	UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:36098 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:50621 [proto: 401/CustomProtocolF][IP: 401/CustomProtocolF][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
-	3	UDP [3ffe:507::1:200:86ff:fe05:80da]:21554 <-> [3ffe:501:4819::42]:5333 [proto: 399/CustomProtocolD][IP: 399/CustomProtocolD][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/90 bytes <-> 1 pkts/510 bytes][Goodput ratio: 31/88][0.07 sec][PLAIN TEXT (itojun)][Plen Bins: 50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	4	UDP [fe80::76ac:b9ff:fe6c:c124]:12717 -> [ff02::1]:64315 [proto: 402/CustomProtocolG][IP: 402/CustomProtocolG][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	5	UDP [fe80::76ac:b9ff:fe6c:c124]:12718 -> [ff02::1]:26993 [proto: 403/CustomProtocolH][IP: 403/CustomProtocolH][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	1	UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:100 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:1991 [proto: 2048/CustomProtocolE][IP: 2048/CustomProtocolE][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No client to server traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
+	2	UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:36098 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:50621 [proto: 2049/CustomProtocolF][IP: 2049/CustomProtocolF][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
+	3	UDP [3ffe:507::1:200:86ff:fe05:80da]:21554 <-> [3ffe:501:4819::42]:5333 [proto: 1024/CustomProtocolD][IP: 1024/CustomProtocolD][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/90 bytes <-> 1 pkts/510 bytes][Goodput ratio: 31/88][0.07 sec][PLAIN TEXT (itojun)][Plen Bins: 50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	4	UDP [fe80::76ac:b9ff:fe6c:c124]:12717 -> [ff02::1]:64315 [proto: 2050/CustomProtocolG][IP: 2050/CustomProtocolG][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	5	UDP [fe80::76ac:b9ff:fe6c:c124]:12718 -> [ff02::1]:26993 [proto: 65535/CustomProtocolH][IP: 65535/CustomProtocolH][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
diff --git a/tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out b/tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out
@@ -1,5 +1,5 @@
 DPI Packets (TCP):	3	(1.00 pkts/flow)
-Confidence Unknown          : 3 (flows)
+Confidence Match by custom rule: 3 (flows)
 Num dissector calls: 0 (0.00 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
@@ -23,11 +23,10 @@ Patricia protocols IPv6: 0/0 (search/found)
 
 CustomProtocolA	3	222	1
 CustomProtocolB	2	148	1
-Unknown	3	222	1
+CustomProtocolC	3	222	1
 
-Acceptable                       5 370           2            
-Unrated                          3 222           1            
+Acceptable                       8 592           3            
 
-	1	TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.396/TLS.CustomProtocolA][IP: 396/CustomProtocolA][Encrypted][Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	2	TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 400/CustomProtocolC][IP: 398/Unknown][ClearText][Confidence: Unknown][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	3	TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 397/CustomProtocolB][IP: 397/CustomProtocolB][ClearText][Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	1	TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.396/TLS.CustomProtocolA][IP: 396/CustomProtocolA][Encrypted][Confidence: Match by custom rule][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	2	TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 800/CustomProtocolC][IP: 800/CustomProtocolC][ClearText][Confidence: Match by custom rule][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	3	TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 397/CustomProtocolB][IP: 397/CustomProtocolB][ClearText][Confidence: Match by custom rule][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]