Skip to content

Commit

Permalink
Improve handling of custom rules
Browse files Browse the repository at this point in the history 
Avoid collisions between user-ids and internal-ids protocols in the
`example/protos.txt` file.
Add a new value for the classification confidence:
`NDPI_CONFIDENCE_CUSTOM_RULE`

With `./example/ndpiReader -p example/protos.txt -H` we now see also the
custom protocols and their internal/external ids:

```
nDPI supported protocols:
 Id Userd-id Protocol               Layer_4    Nw_Proto Breed        Category
  0        0 Unknown                TCP        X        Unrated      Unspecified

...

387      387 Mumble                 UDP        X        Fun          VoIP
388      388 iSCSI                  TCP                 Acceptable   Unspecified
389      389 Kibana                 TCP                 Acceptable   Unspecified
390      390 TestProto              TCP                 Acceptable   Unspecified
391      391 HomeRouter             TCP                 Acceptable   Unspecified
392      392 CustomProtocol         TCP                 Acceptable   Unspecified
393      393 AmazonPrime            TCP                 Acceptable   Unspecified
394      394 CustomProtocolA        TCP                 Acceptable   Unspecified
395      395 CustomProtocolB        TCP                 Acceptable   Unspecified
396      800 CustomProtocolC        TCP                 Acceptable   Unspecified
397     1024 CustomProtocolD        TCP                 Acceptable   Unspecified
398     2048 CustomProtocolE        TCP                 Acceptable   Unspecified
399     2049 CustomProtocolF        TCP                 Acceptable   Unspecified
400     2050 CustomProtocolG        TCP                 Acceptable   Unspecified
401    65535 CustomProtocolH        TCP                 Acceptable   Unspecified
```

We likely need to take a better look in general at the iteration between
internal and external protocols ids...

This PR fixes the issue observed in
#2274 (comment) and in
#2275.
  • Loading branch information
@IvanNardi
IvanNardi committed Jan 21, 2024
1 parent 49591dd commit 450ec94
Show file tree
Hide file tree
Showing 7 changed files with 50 additions and 34 deletions.
14 changes: 10 additions & 4 deletions example/ndpiReader.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -609,6 +609,10 @@ static void help(u_int long_help) {
struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module();
NDPI_BITMASK_SET_ALL(all);
ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all);

if(_protoFilePath != NULL)
ndpi_load_protocols_file(ndpi_info_mod, _protoFilePath);

ndpi_finalize_initialization(ndpi_info_mod);

printf("\nProtocols configuration parameters:\n");
Expand All @@ -635,8 +639,8 @@ static void help(u_int long_help) {
sizeof(((struct ndpi_flow_struct *)0)->protos));

printf("\n\nnDPI supported protocols:\n");
printf("%3s %-22s %-10s %-8s %-12s %s\n",
"Id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category");
printf("%3s %8s %-22s %-10s %-8s %-12s %s\n",
"Id", "Userd-id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category");
num_threads = 1;

ndpi_dump_protocols(ndpi_info_mod, stdout);
Expand Down Expand Up @@ -4126,7 +4130,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us

if(!quiet_mode) printf("\n\nDetected protocols:\n");
for(i = 0; i <= ndpi_get_num_supported_protocols(ndpi_thread_info[0].workflow->ndpi_struct); i++) {
ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct, i);
ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct,
ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i));

if(cumulative_stats.protocol_counter[i] > 0) {
breed_stats_bytes[breed] += (long long unsigned int)cumulative_stats.protocol_counter_bytes[i];
Expand All @@ -4135,7 +4140,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us

if(results_file)
fprintf(results_file, "%s\t%llu\t%llu\t%u\n",
ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct, i),
ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct,
ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i)),
(long long unsigned int)cumulative_stats.protocol_counter[i],
(long long unsigned int)cumulative_stats.protocol_counter_bytes[i],
cumulative_stats.protocol_flows[i]);
Expand Down
15 changes: 9 additions & 6 deletions example/protos.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,18 @@ ip:213.75.170.11/32:443@CustomProtocol
ip:8.248.73.247:443@AmazonPrime
ip:54.80.47.130@AmazonPrime

#You can specify a protocol Id. In that case you probably want to avoid conflict with internal ids.
#You can use any number up to 65535

ip:3.3.3.3:443@CustomProtocolA
ip:3.3.3.3:444@CustomProtocolB
ip:3.3.3.3:446@CustomProtocolC=400
ip:3.3.3.3:446@CustomProtocolC=800

ipv6:[3ffe:507:0:1:200:86ff:fe05:80da]@CustomProtocolD
ipv6:[247f:855b:5e16:3caf::]/64:100@CustomProtocolE
ipv6:[247f:855b:5e16:3caf::]/64@CustomProtocolF
ipv6:[fe80::76ac:b9ff:fe6c:c124]:12717@CustomProtocolG
ipv6:[fe80::76ac:b9ff:fe6c:c124]:12718@CustomProtocolH
ipv6:[3ffe:507:0:1:200:86ff:fe05:80da]@CustomProtocolD=1024
ipv6:[247f:855b:5e16:3caf::]/64:100@CustomProtocolE=2048
ipv6:[247f:855b:5e16:3caf::]/64@CustomProtocolF=2049
ipv6:[fe80::76ac:b9ff:fe6c:c124]:12717@CustomProtocolG=2050
ipv6:[fe80::76ac:b9ff:fe6c:c124]:12718@CustomProtocolH=65535

#
# You can use symbolic IP addreses if you want
Expand Down
1 change: 1 addition & 0 deletions src/include/ndpi_typedefs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -958,6 +958,7 @@ typedef enum {
NDPI_CONFIDENCE_DPI, /* Deep packet inspection */
NDPI_CONFIDENCE_MATCH_BY_IP, /* Classification obtained looking only at the IP addresses */
NDPI_CONFIDENCE_DPI_AGGRESSIVE, /* Aggressive DPI: it might be a false positive */
NDPI_CONFIDENCE_CUSTOM_RULE, /* Matching a custom rules */

/*
IMPORTANT
Expand Down
17 changes: 12 additions & 5 deletions src/lib/ndpi_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7932,7 +7932,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
/* This is a custom protocol and it has priority over everything else */
ret->master_protocol = NDPI_PROTOCOL_UNKNOWN,
ret->app_protocol = flow->guessed_protocol_id;
flow->confidence = NDPI_CONFIDENCE_MATCH_BY_PORT; /* TODO */
flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
ndpi_fill_protocol_category(ndpi_str, flow, ret);
return(-1);
}
Expand All @@ -7944,6 +7944,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
*ret = ndpi_detection_giveup(ndpi_str, flow, &protocol_was_guessed);
}

flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
ndpi_fill_protocol_category(ndpi_str, flow, ret);
return(-1);
}
Expand All @@ -7957,6 +7958,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n

flow->num_dissector_calls += ndpi_check_flow_func(ndpi_str, flow, &ndpi_selection_packet);

flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
ndpi_fill_protocol_category(ndpi_str, flow, ret);
return(-1);
}
Expand Down Expand Up @@ -8281,8 +8283,9 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
packetlen, current_time_ms,
input_info);

p.master_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.master_protocol),
p.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.app_protocol);
p.master_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.master_protocol);
p.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.app_protocol);
p.protocol_by_ip = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.protocol_by_ip);

return(p);
}
Expand Down Expand Up @@ -9166,6 +9169,9 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)
case NDPI_CONFIDENCE_DPI_AGGRESSIVE:
return "DPI (aggressive)";

case NDPI_CONFIDENCE_CUSTOM_RULE:
return "Match by custom rule";

default:
return NULL;
}
Expand Down Expand Up @@ -9346,8 +9352,9 @@ void ndpi_dump_protocols(struct ndpi_detection_module_struct *ndpi_str, FILE *du
if(!ndpi_str || !dump_out) return;

for(i = 0; i < (int) ndpi_str->ndpi_num_supported_protocols; i++)
fprintf(dump_out, "%3d %-22s %-10s %-8s %-12s %s\n",
i, ndpi_str->proto_defaults[i].protoName,
fprintf(dump_out, "%3d %8d %-22s %-10s %-8s %-12s %s\n",
i, ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, i),
ndpi_str->proto_defaults[i].protoName,
ndpi_get_l4_proto_name(ndpi_get_l4_proto_info(ndpi_str, i)),
ndpi_str->proto_defaults[i].isAppProtocol ? "" : "X",
ndpi_get_proto_breed_name(ndpi_str, ndpi_str->proto_defaults[i].protoBreed),
Expand Down
17 changes: 8 additions & 9 deletions tests/cfgs/default/result/custom_rules_ipv6.pcapng.out
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
DPI Packets (UDP): 5 (1.00 pkts/flow)
Confidence Unknown : 5 (flows)
Confidence Match by custom rule: 5 (flows)
Num dissector calls: 0 (0.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
Expand All @@ -21,17 +21,16 @@ Patricia risk IPv6: 0/0 (search/found)
Patricia protocols: 0/0 (search/found)
Patricia protocols IPv6: 9/5 (search/found)

Unknown 1 318 1
CustomProtocolD 2 600 1
CustomProtocolE 1 1287 1
CustomProtocolF 1 1287 1
CustomProtocolG 1 318 1
CustomProtocolH 1 318 1

Acceptable 5 3492 4
Unrated 1 318 1
Acceptable 6 3810 5

1 UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:100 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:1991 [proto: 398/CustomProtocolE][IP: 398/CustomProtocolE][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No client to server traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
2 UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:36098 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:50621 [proto: 399/CustomProtocolF][IP: 399/CustomProtocolF][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
3 UDP [3ffe:507::1:200:86ff:fe05:80da]:21554 <-> [3ffe:501:4819::42]:5333 [proto: 397/CustomProtocolD][IP: 397/CustomProtocolD][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/90 bytes <-> 1 pkts/510 bytes][Goodput ratio: 31/88][0.07 sec][PLAIN TEXT (itojun)][Plen Bins: 50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 UDP [fe80::76ac:b9ff:fe6c:c124]:12717 -> [ff02::1]:64315 [proto: 400/CustomProtocolC][IP: 400/CustomProtocolC][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
5 UDP [fe80::76ac:b9ff:fe6c:c124]:12718 -> [ff02::1]:26993 [proto: 401/CustomProtocolH][IP: 401/CustomProtocolH][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:100 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:1991 [proto: 2048/CustomProtocolE][IP: 2048/CustomProtocolE][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No client to server traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
2 UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:36098 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:50621 [proto: 2049/CustomProtocolF][IP: 2049/CustomProtocolF][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
3 UDP [3ffe:507::1:200:86ff:fe05:80da]:21554 <-> [3ffe:501:4819::42]:5333 [proto: 1024/CustomProtocolD][IP: 1024/CustomProtocolD][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/90 bytes <-> 1 pkts/510 bytes][Goodput ratio: 31/88][0.07 sec][PLAIN TEXT (itojun)][Plen Bins: 50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 UDP [fe80::76ac:b9ff:fe6c:c124]:12717 -> [ff02::1]:64315 [proto: 2050/CustomProtocolG][IP: 2050/CustomProtocolG][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
5 UDP [fe80::76ac:b9ff:fe6c:c124]:12718 -> [ff02::1]:26993 [proto: 65535/CustomProtocolH][IP: 65535/CustomProtocolH][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
13 changes: 6 additions & 7 deletions tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
DPI Packets (TCP): 3 (1.00 pkts/flow)
Confidence Unknown : 3 (flows)
Confidence Match by custom rule: 3 (flows)
Num dissector calls: 0 (0.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
Expand All @@ -23,11 +23,10 @@ Patricia protocols IPv6: 0/0 (search/found)

CustomProtocolA 3 222 1
CustomProtocolB 2 148 1
Unknown 3 222 1
CustomProtocolC 3 222 1

Acceptable 5 370 2
Unrated 3 222 1
Acceptable 8 592 3

1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.394/TLS.CustomProtocolA][IP: 394/CustomProtocolA][Encrypted][Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 400/CustomProtocolC][IP: 396/Unknown][ClearText][Confidence: Unknown][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 395/CustomProtocolB][IP: 395/CustomProtocolB][ClearText][Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.394/TLS.CustomProtocolA][IP: 394/CustomProtocolA][Encrypted][Confidence: Match by custom rule][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 800/CustomProtocolC][IP: 800/CustomProtocolC][ClearText][Confidence: Match by custom rule][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 395/CustomProtocolB][IP: 395/CustomProtocolB][ClearText][Confidence: Match by custom rule][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Loading

0 comments on commit 450ec94

Please sign in to comment.