Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve handling of custom rules #2276

Merged
merged 1 commit into from
Jan 21, 2024
Merged

Improve handling of custom rules #2276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions example/ndpiReader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -609,6 +609,10 @@ static void help(u_int long_help) {
struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module();
NDPI_BITMASK_SET_ALL(all);
ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all);

if(_protoFilePath != NULL)
ndpi_load_protocols_file(ndpi_info_mod, _protoFilePath);

ndpi_finalize_initialization(ndpi_info_mod);

printf("\nProtocols configuration parameters:\n");
Expand All @@ -635,8 +639,8 @@ static void help(u_int long_help) {
sizeof(((struct ndpi_flow_struct *)0)->protos));

printf("\n\nnDPI supported protocols:\n");
printf("%3s %-22s %-10s %-8s %-12s %s\n",
"Id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category");
printf("%3s %8s %-22s %-10s %-8s %-12s %s\n",
"Id", "Userd-id", "Protocol", "Layer_4", "Nw_Proto", "Breed", "Category");
num_threads = 1;

ndpi_dump_protocols(ndpi_info_mod, stdout);
Expand Down Expand Up @@ -4126,7 +4130,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us

if(!quiet_mode) printf("\n\nDetected protocols:\n");
for(i = 0; i <= ndpi_get_num_supported_protocols(ndpi_thread_info[0].workflow->ndpi_struct); i++) {
ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct, i);
ndpi_protocol_breed_t breed = ndpi_get_proto_breed(ndpi_thread_info[0].workflow->ndpi_struct,
ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i));

if(cumulative_stats.protocol_counter[i] > 0) {
breed_stats_bytes[breed] += (long long unsigned int)cumulative_stats.protocol_counter_bytes[i];
Expand All @@ -4135,7 +4140,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us

if(results_file)
fprintf(results_file, "%s\t%llu\t%llu\t%u\n",
ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct, i),
ndpi_get_proto_name(ndpi_thread_info[0].workflow->ndpi_struct,
ndpi_map_ndpi_id_to_user_proto_id(ndpi_thread_info[0].workflow->ndpi_struct, i)),
(long long unsigned int)cumulative_stats.protocol_counter[i],
(long long unsigned int)cumulative_stats.protocol_counter_bytes[i],
cumulative_stats.protocol_flows[i]);
Expand Down
15 changes: 9 additions & 6 deletions example/protos.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,18 @@ ip:213.75.170.11/32:443@CustomProtocol
ip:8.248.73.247:443@AmazonPrime
ip:54.80.47.130@AmazonPrime

#You can specify a protocol Id. In that case you probably want to avoid conflict with internal ids.
#You can use any number up to 65535

ip:3.3.3.3:443@CustomProtocolA
ip:3.3.3.3:444@CustomProtocolB
ip:3.3.3.3:446@CustomProtocolC=400
ip:3.3.3.3:446@CustomProtocolC=800

ipv6:[3ffe:507:0:1:200:86ff:fe05:80da]@CustomProtocolD
ipv6:[247f:855b:5e16:3caf::]/64:100@CustomProtocolE
ipv6:[247f:855b:5e16:3caf::]/64@CustomProtocolF
ipv6:[fe80::76ac:b9ff:fe6c:c124]:12717@CustomProtocolG
ipv6:[fe80::76ac:b9ff:fe6c:c124]:12718@CustomProtocolH
ipv6:[3ffe:507:0:1:200:86ff:fe05:80da]@CustomProtocolD=1024
ipv6:[247f:855b:5e16:3caf::]/64:100@CustomProtocolE=2048
ipv6:[247f:855b:5e16:3caf::]/64@CustomProtocolF=2049
ipv6:[fe80::76ac:b9ff:fe6c:c124]:12717@CustomProtocolG=2050
ipv6:[fe80::76ac:b9ff:fe6c:c124]:12718@CustomProtocolH=65535

#
# You can use symbolic IP addreses if you want
Expand Down
1 change: 1 addition & 0 deletions src/include/ndpi_typedefs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -958,6 +958,7 @@ typedef enum {
NDPI_CONFIDENCE_DPI, /* Deep packet inspection */
NDPI_CONFIDENCE_MATCH_BY_IP, /* Classification obtained looking only at the IP addresses */
NDPI_CONFIDENCE_DPI_AGGRESSIVE, /* Aggressive DPI: it might be a false positive */
NDPI_CONFIDENCE_CUSTOM_RULE, /* Matching a custom rules */

/*
IMPORTANT
Expand Down
17 changes: 12 additions & 5 deletions src/lib/ndpi_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7939,7 +7939,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
/* This is a custom protocol and it has priority over everything else */
ret->master_protocol = NDPI_PROTOCOL_UNKNOWN,
ret->app_protocol = flow->guessed_protocol_id;
flow->confidence = NDPI_CONFIDENCE_MATCH_BY_PORT; /* TODO */
flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
ndpi_fill_protocol_category(ndpi_str, flow, ret);
return(-1);
}
Expand All @@ -7951,6 +7951,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
*ret = ndpi_detection_giveup(ndpi_str, flow, &protocol_was_guessed);
}

flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
ndpi_fill_protocol_category(ndpi_str, flow, ret);
return(-1);
}
Expand All @@ -7964,6 +7965,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n

flow->num_dissector_calls += ndpi_check_flow_func(ndpi_str, flow, &ndpi_selection_packet);

flow->confidence = NDPI_CONFIDENCE_CUSTOM_RULE;
ndpi_fill_protocol_category(ndpi_str, flow, ret);
return(-1);
}
Expand Down Expand Up @@ -8288,8 +8290,9 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
packetlen, current_time_ms,
input_info);

p.master_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.master_protocol),
p.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.app_protocol);
p.master_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.master_protocol);
p.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.app_protocol);
p.protocol_by_ip = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.protocol_by_ip);

return(p);
}
Expand Down Expand Up @@ -9173,6 +9176,9 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)
case NDPI_CONFIDENCE_DPI_AGGRESSIVE:
return "DPI (aggressive)";

case NDPI_CONFIDENCE_CUSTOM_RULE:
return "Match by custom rule";

default:
return NULL;
}
Expand Down Expand Up @@ -9353,8 +9359,9 @@ void ndpi_dump_protocols(struct ndpi_detection_module_struct *ndpi_str, FILE *du
if(!ndpi_str || !dump_out) return;

for(i = 0; i < (int) ndpi_str->ndpi_num_supported_protocols; i++)
fprintf(dump_out, "%3d %-22s %-10s %-8s %-12s %s\n",
i, ndpi_str->proto_defaults[i].protoName,
fprintf(dump_out, "%3d %8d %-22s %-10s %-8s %-12s %s\n",
i, ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, i),
ndpi_str->proto_defaults[i].protoName,
ndpi_get_l4_proto_name(ndpi_get_l4_proto_info(ndpi_str, i)),
ndpi_str->proto_defaults[i].isAppProtocol ? "" : "X",
ndpi_get_proto_breed_name(ndpi_str, ndpi_str->proto_defaults[i].protoBreed),
Expand Down
17 changes: 8 additions & 9 deletions tests/cfgs/default/result/custom_rules_ipv6.pcapng.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
DPI Packets (UDP): 5 (1.00 pkts/flow)
Confidence Unknown : 5 (flows)
Confidence Match by custom rule: 5 (flows)
Num dissector calls: 0 (0.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
Expand All @@ -21,17 +21,16 @@ Patricia risk IPv6: 0/0 (search/found)
Patricia protocols: 0/0 (search/found)
Patricia protocols IPv6: 9/5 (search/found)

Unknown 1 1287 1
CustomProtocolD 2 600 1
CustomProtocolE 1 1287 1
CustomProtocolF 1 1287 1
CustomProtocolG 1 318 1
CustomProtocolH 1 318 1

Acceptable 5 2523 4
Unrated 1 1287 1
Acceptable 6 3810 5

1 UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:100 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:1991 [proto: 400/CustomProtocolC][IP: 400/CustomProtocolC][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No client to server traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
2 UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:36098 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:50621 [proto: 401/CustomProtocolF][IP: 401/CustomProtocolF][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
3 UDP [3ffe:507::1:200:86ff:fe05:80da]:21554 <-> [3ffe:501:4819::42]:5333 [proto: 399/CustomProtocolD][IP: 399/CustomProtocolD][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/90 bytes <-> 1 pkts/510 bytes][Goodput ratio: 31/88][0.07 sec][PLAIN TEXT (itojun)][Plen Bins: 50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 UDP [fe80::76ac:b9ff:fe6c:c124]:12717 -> [ff02::1]:64315 [proto: 402/CustomProtocolG][IP: 402/CustomProtocolG][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
5 UDP [fe80::76ac:b9ff:fe6c:c124]:12718 -> [ff02::1]:26993 [proto: 403/CustomProtocolH][IP: 403/CustomProtocolH][ClearText][Confidence: Unknown][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:100 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:1991 [proto: 2048/CustomProtocolE][IP: 2048/CustomProtocolE][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No client to server traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
2 UDP [247f:855b:5e16:3caf:3f2c:4134:9592:661b]:36098 -> [21bc:b273:7f68:88d7:77a8:585:3990:927b]:50621 [proto: 2049/CustomProtocolF][IP: 2049/CustomProtocolF][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/1287 bytes -> 0 pkts/0 bytes][Goodput ratio: 95/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0]
3 UDP [3ffe:507::1:200:86ff:fe05:80da]:21554 <-> [3ffe:501:4819::42]:5333 [proto: 1024/CustomProtocolD][IP: 1024/CustomProtocolD][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/90 bytes <-> 1 pkts/510 bytes][Goodput ratio: 31/88][0.07 sec][PLAIN TEXT (itojun)][Plen Bins: 50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 UDP [fe80::76ac:b9ff:fe6c:c124]:12717 -> [ff02::1]:64315 [proto: 2050/CustomProtocolG][IP: 2050/CustomProtocolG][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
5 UDP [fe80::76ac:b9ff:fe6c:c124]:12718 -> [ff02::1]:26993 [proto: 65535/CustomProtocolH][IP: 65535/CustomProtocolH][ClearText][Confidence: Match by custom rule][DPI packets: 1][1 pkts/318 bytes -> 0 pkts/0 bytes][Goodput ratio: 80/0][< 1 sec][PLAIN TEXT (BZ.qca956)][Plen Bins: 0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
13 changes: 6 additions & 7 deletions tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
DPI Packets (TCP): 3 (1.00 pkts/flow)
Confidence Unknown : 3 (flows)
Confidence Match by custom rule: 3 (flows)
Num dissector calls: 0 (0.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
Expand All @@ -23,11 +23,10 @@ Patricia protocols IPv6: 0/0 (search/found)

CustomProtocolA 3 222 1
CustomProtocolB 2 148 1
Unknown 3 222 1
CustomProtocolC 3 222 1

Acceptable 5 370 2
Unrated 3 222 1
Acceptable 8 592 3

1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.396/TLS.CustomProtocolA][IP: 396/CustomProtocolA][Encrypted][Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 400/CustomProtocolC][IP: 398/Unknown][ClearText][Confidence: Unknown][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 397/CustomProtocolB][IP: 397/CustomProtocolB][ClearText][Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.396/TLS.CustomProtocolA][IP: 396/CustomProtocolA][Encrypted][Confidence: Match by custom rule][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 800/CustomProtocolC][IP: 800/CustomProtocolC][ClearText][Confidence: Match by custom rule][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 397/CustomProtocolB][IP: 397/CustomProtocolB][ClearText][Confidence: Match by custom rule][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Loading
Loading