Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Replaced ndpiReader's libjson-c support with libnDPI's internal serialization interface. #1535

Merged
merged 1 commit into from
May 7, 2022
Merged

Replaced ndpiReader's libjson-c support with libnDPI's internal serialization interface. #1535

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,11 @@ AC_DEFINE_UNQUOTED(NDPI_GIT_DATE, "${GIT_DATE}", [Last GIT change])

if ! test "${with_only_libndpi+set}" = set; then :
dnl> used by json-c for unit tests
PKG_CHECK_MODULES([JSONC], [json-c], [JSONC_LIBS="${pkg_cv_JSONC_LIBS}" JSONC_CFLAGS="${pkg_cv_JSONC_CFLAGS}"], [AC_MSG_WARN([JSON-C not available. Disabled unit test.])])
PKG_CHECK_MODULES([JSONC], [json-c], [
AC_DEFINE(HAVE_LIBJSON_C, 1, [libjson-c is present])
JSONC_LIBS="${pkg_cv_JSONC_LIBS}"
JSONC_CFLAGS="${pkg_cv_JSONC_CFLAGS}"
], [AC_MSG_WARN([JSON-C not available. Disabled unit test.])])
AC_CHECK_LIB([json-c], [json_object_put], [EXTRA_TARGETS="$EXTRA_TARGETS tests/unit"], [
AC_MSG_WARN([JSON-C not available. Disabled unit test.])
JSONC_LIBS=""
Expand Down
1,538 changes: 474 additions & 1,064 deletions example/ndpiReader.c
View file
Open in desktop

Large diffs are not rendered by default.

113 changes: 61 additions & 52 deletions example/reader_util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,8 @@ extern char *_debug_protocols;
static int _debug_protocols_ok = 0;

struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs,
pcap_t * pcap_handle, int do_init_flows_root) {
pcap_t * pcap_handle, int do_init_flows_root,
ndpi_serialization_format serialization_format) {
struct ndpi_detection_module_struct * module;
struct ndpi_workflow * workflow;

Expand Down Expand Up @@ -433,6 +434,14 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
if(do_init_flows_root)
workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *));

if (serialization_format != ndpi_serialization_format_unknown &&
ndpi_init_serializer(&workflow->ndpi_serializer,
serialization_format) != 0)
{
LOG(NDPI_LOG_ERROR, "serializer initialization failed\n");
exit(-1);
}

return workflow;
}

Expand Down Expand Up @@ -539,6 +548,8 @@ void ndpi_flow_info_free_data(struct ndpi_flow_info *flow) {
void ndpi_workflow_free(struct ndpi_workflow * workflow) {
u_int i;

ndpi_term_serializer(&workflow->ndpi_serializer);

for(i=0; i<workflow->prefs.num_roots; i++)
ndpi_tdestroy(workflow->ndpi_flows_root[i], ndpi_flow_info_freer);

Expand Down Expand Up @@ -857,12 +868,14 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
inet_ntop(AF_INET, &newflow->src_ip, newflow->src_name, sizeof(newflow->src_name));
inet_ntop(AF_INET, &newflow->dst_ip, newflow->dst_name, sizeof(newflow->dst_name));
} else {
struct in6_addr addr = *(struct in6_addr *)&iph6->ip6_src;
inet_ntop(AF_INET6, &addr, newflow->src_name, sizeof(newflow->src_name));
addr = *(struct in6_addr *)&iph6->ip6_dst;
inet_ntop(AF_INET6, &addr, newflow->dst_name, sizeof(newflow->dst_name));
/* For consistency across platforms replace :0: with :: */
ndpi_patchIPv6Address(newflow->src_name), ndpi_patchIPv6Address(newflow->dst_name);
newflow->src_ip6 = *(struct ndpi_in6_addr *)&iph6->ip6_src;
inet_ntop(AF_INET6, &newflow->src_ip6,
newflow->src_name, sizeof(newflow->src_name));
newflow->dst_ip6 = *(struct ndpi_in6_addr *)&iph6->ip6_dst;
inet_ntop(AF_INET6, &newflow->dst_ip6,
newflow->dst_name, sizeof(newflow->dst_name));
/* For consistency across platforms replace :0: with :: */
ndpi_patchIPv6Address(newflow->src_name), ndpi_patchIPv6Address(newflow->dst_name);
}

if((newflow->ndpi_flow = ndpi_flow_malloc(SIZEOF_FLOW_STRUCT)) == NULL) {
Expand Down Expand Up @@ -1021,11 +1034,13 @@ u_int8_t plen2slot(u_int16_t plen) {

/* ****************************************************** */

void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow, FILE * csv_fp) {
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) {
u_int i, is_quic = 0;

if(!flow->ndpi_flow) return;

flow->info_type = INFO_INVALID;

flow->confidence = flow->ndpi_flow->confidence;

ndpi_snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s",
Expand Down Expand Up @@ -1059,8 +1074,11 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
/* DNS */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_DNS)) {
if(flow->ndpi_flow->protos.dns.rsp_type == 0x1)
{
flow->info_type = INFO_GENERIC;
inet_ntop(AF_INET, &flow->ndpi_flow->protos.dns.rsp_addr.ipv4, flow->info, sizeof(flow->info));
else {
} else {
flow->info_type = INFO_GENERIC;
inet_ntop(AF_INET6, &flow->ndpi_flow->protos.dns.rsp_addr.ipv6, flow->info, sizeof(flow->info));

/* For consistency across platforms replace :0: with :: */
Expand All @@ -1069,44 +1087,41 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
}
/* MDNS */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_MDNS)) {
flow->info_type = INFO_GENERIC;
ndpi_snprintf(flow->info, sizeof(flow->info), "%s", flow->ndpi_flow->host_server_name);
}
/* UBNTAC2 */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_UBNTAC2)) {
flow->info_type = INFO_GENERIC;
ndpi_snprintf(flow->info, sizeof(flow->info), "%s", flow->ndpi_flow->protos.ubntac2.version);
}
/* FTP */
else if((is_ndpi_proto(flow, NDPI_PROTOCOL_FTP_CONTROL))
|| /* IMAP */ is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_IMAP)
|| /* POP */ is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_POP)
|| /* SMTP */ is_ndpi_proto(flow, NDPI_PROTOCOL_MAIL_SMTP)) {
if(flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username[0] != '\0')
ndpi_snprintf(flow->info, sizeof(flow->info), "User: %s][Pwd: %s%s",
flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username,
flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password,
flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed ? "][Auth Failed" : "");
flow->info_type = INFO_FTP_IMAP_POP_SMTP;
ndpi_snprintf(flow->ftp_imap_pop_smtp.username,
sizeof(flow->ftp_imap_pop_smtp.username),
"%s", flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.username);
ndpi_snprintf(flow->ftp_imap_pop_smtp.password,
sizeof(flow->ftp_imap_pop_smtp.password),
"%s", flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.password);
flow->ftp_imap_pop_smtp.auth_failed =
flow->ndpi_flow->l4.tcp.ftp_imap_pop_smtp.auth_failed;
}
/* KERBEROS */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_KERBEROS)) {
if((flow->ndpi_flow->protos.kerberos.hostname[0] != '\0')
|| (flow->ndpi_flow->protos.kerberos.username[0] != '\0')) {
ndpi_snprintf(flow->info, sizeof(flow->info), "%s%s%s%s",
flow->ndpi_flow->protos.kerberos.domain /* = realm */,
flow->ndpi_flow->protos.kerberos.domain[0] != '\0' ? "\\" : "",
flow->ndpi_flow->protos.kerberos.hostname,
flow->ndpi_flow->protos.kerberos.username);
} else if(flow->ndpi_flow->protos.kerberos.domain[0] != '\0')
ndpi_snprintf(flow->info, sizeof(flow->info), "%s",
flow->ndpi_flow->protos.kerberos.domain);

#if 0
if(flow->info[0] != '\0')
printf("->> (%d) [%s][%s][%s]<<--\n",
htons(flow->src_port),
flow->ndpi_flow->protos.kerberos.domain,
flow->ndpi_flow->protos.kerberos.hostname,
flow->ndpi_flow->protos.kerberos.username);
#endif
flow->info_type = INFO_KERBEROS;
ndpi_snprintf(flow->kerberos.domain,
sizeof(flow->kerberos.domain),
"%s", flow->ndpi_flow->protos.kerberos.domain);
ndpi_snprintf(flow->kerberos.hostname,
sizeof(flow->kerberos.hostname),
"%s", flow->ndpi_flow->protos.kerberos.hostname);
ndpi_snprintf(flow->kerberos.username,
sizeof(flow->kerberos.username),
"%s", flow->ndpi_flow->protos.kerberos.username);
}
/* HTTP */
else if((flow->detected_protocol.master_protocol == NDPI_PROTOCOL_HTTP)
Expand Down Expand Up @@ -1187,22 +1202,18 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
correct_csv_data_field(flow->ndpi_flow->protos.tls_quic.alpn);
correct_csv_data_field(flow->ndpi_flow->protos.tls_quic.tls_supported_versions);

if(csv_fp)
ndpi_snprintf(flow->info, sizeof(flow->info), "%s",
flow->ndpi_flow->protos.tls_quic.alpn);
else
ndpi_snprintf(flow->info, sizeof(flow->info), "ALPN: %s][TLS Supported Versions: %s",
flow->ndpi_flow->protos.tls_quic.alpn,
flow->ndpi_flow->protos.tls_quic.tls_supported_versions);
flow->info_type = INFO_TLS_QUIC_ALPN_VERSION;
ndpi_snprintf(flow->tls_quic.alpn, sizeof(flow->tls_quic.alpn), "%s",
flow->ndpi_flow->protos.tls_quic.alpn);
ndpi_snprintf(flow->tls_quic.tls_supported_versions,
sizeof(flow->tls_quic.tls_supported_versions),
"%s", flow->ndpi_flow->protos.tls_quic.tls_supported_versions);
} else if(flow->ndpi_flow->protos.tls_quic.alpn) {
correct_csv_data_field(flow->ndpi_flow->protos.tls_quic.alpn);

if(csv_fp)
ndpi_snprintf(flow->info, sizeof(flow->info), "%s,",
flow->ndpi_flow->protos.tls_quic.alpn);
else
ndpi_snprintf(flow->info, sizeof(flow->info), "ALPN: %s",
flow->ndpi_flow->protos.tls_quic.alpn);
flow->info_type = INFO_TLS_QUIC_ALPN_ONLY;
ndpi_snprintf(flow->tls_quic.alpn, sizeof(flow->tls_quic.alpn), "%s",
flow->ndpi_flow->protos.tls_quic.alpn);
}

if(enable_doh_dot_detection) {
Expand Down Expand Up @@ -1305,8 +1316,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
const struct pcap_pkthdr *header,
const u_char *packet,
pkt_timeval when,
ndpi_risk *flow_risk,
FILE * csv_fp) {
ndpi_risk *flow_risk) {
struct ndpi_flow_info *flow = NULL;
struct ndpi_flow_struct *ndpi_flow = NULL;
u_int8_t proto;
Expand Down Expand Up @@ -1546,7 +1556,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if(enable_protocol_guess) workflow->stats.guessed_flow_protocols++;
}

process_ndpi_collected_info(workflow, flow, csv_fp);
process_ndpi_collected_info(workflow, flow);
}
}
}
Expand Down Expand Up @@ -1598,8 +1608,7 @@ int ndpi_is_datalink_supported(int datalink_type) {
struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
const struct pcap_pkthdr *header,
const u_char *packet,
ndpi_risk *flow_risk,
FILE * csv_fp) {
ndpi_risk *flow_risk) {
/*
* Declare pointers to packet headers
*/
Expand Down Expand Up @@ -2093,7 +2102,7 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
return(packet_processing(workflow, time_ms, vlan_id, tunnel_type, iph, iph6,
ip_offset, header->caplen - ip_offset,
header->caplen, header, packet, header->ts,
flow_risk, csv_fp));
flow_risk));
}

/* ********************************************************** */
Expand Down
42 changes: 37 additions & 5 deletions example/reader_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ extern int dpdk_port_deinit(int port);
#define MAX_TABLE_SIZE_1 4096
#define MAX_TABLE_SIZE_2 8192
#define INIT_VAL -1
#define SERIALIZATION_BUFSIZ (8192 * 2)


// inner hash table (ja3 -> security state)
Expand Down Expand Up @@ -157,12 +158,23 @@ struct ndpi_entropy {
float score;
};

enum info_type {
INFO_INVALID = 0,
INFO_GENERIC,
INFO_KERBEROS,
INFO_FTP_IMAP_POP_SMTP,
INFO_TLS_QUIC_ALPN_VERSION,
INFO_TLS_QUIC_ALPN_ONLY,
};

// flow tracking
typedef struct ndpi_flow_info {
u_int32_t flow_id;
u_int32_t hashval;
u_int32_t src_ip; /* network order */
u_int32_t dst_ip; /* network order */
struct ndpi_in6_addr src_ip6; /* network order */
struct ndpi_in6_addr dst_ip6; /* network order */
u_int16_t src_port; /* network order */
u_int16_t dst_port; /* network order */
u_int8_t detection_completed, protocol, bidirectional, check_extra_packets;
Expand Down Expand Up @@ -196,7 +208,25 @@ typedef struct ndpi_flow_info {
struct ndpi_analyze_struct *iat_c_to_s, *iat_s_to_c, *iat_flow,
*pktlen_c_to_s, *pktlen_s_to_c;

char info[255];
enum info_type info_type;
union {
char info[256];
struct {
char alpn[128];
char tls_supported_versions[128];
} tls_quic;
struct {
unsigned char auth_failed;
char username[127];
char password[128];
} ftp_imap_pop_smtp;
struct {
char domain[85];
char hostname[85];
char username[86];
} kerberos;
};

char flow_extra_info[16];
char host_server_name[80]; /* Hostname/SNI */
char *bittorent_hash;
Expand Down Expand Up @@ -302,11 +332,14 @@ typedef struct ndpi_workflow {
void **ndpi_flows_root;
struct ndpi_detection_module_struct *ndpi_struct;
u_int32_t num_allocated_flows;

/* CSV,TLV,JSON serialization interface */
ndpi_serializer ndpi_serializer;
} ndpi_workflow_t;


/* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */
struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle, int do_init_flows_root);
struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle, int do_init_flows_root, ndpi_serialization_format serialization_format);


/* workflow main free function */
Expand All @@ -324,8 +357,7 @@ void ndpi_free_flow_info_half(struct ndpi_flow_info *flow);
struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
const struct pcap_pkthdr *header,
const u_char *packet,
ndpi_risk *flow_risk,
FILE * csv_fp);
ndpi_risk *flow_risk);

int ndpi_is_datalink_supported(int datalink_type);

Expand All @@ -345,7 +377,7 @@ static inline void ndpi_workflow_set_flow_giveup_callback(struct ndpi_workflow *

/* compare two nodes in workflow */
int ndpi_workflow_node_cmp(const void *a, const void *b);
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow, FILE * csv_fp);
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow);
u_int32_t ethernet_crc32(const void* data, size_t n_bytes);
void ndpi_flow_info_free_data(struct ndpi_flow_info *flow);
void ndpi_flow_info_freer(void *node);
Expand Down
6 changes: 4 additions & 2 deletions fuzz/fuzz_ndpi_reader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
prefs->max_ndpi_flows = 1024 * 1024;
prefs->quiet_mode = 0;

workflow = ndpi_workflow_init(prefs, NULL /* pcap handler will be set later */, 0);
workflow = ndpi_workflow_init(prefs, NULL /* pcap handler will be set later */, 0, ndpi_serialization_format_json);
// enable all protocols
NDPI_BITMASK_SET_ALL(all);
ndpi_set_protocol_detection_bitmask2(workflow->ndpi_struct, &all);
Expand All @@ -79,6 +79,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
if (pkts == NULL) {
remove(pcap_path);
free(pcap_path);
ndpi_term_serializer(&workflow->ndpi_serializer);
return 0;
}
if (ndpi_is_datalink_supported(pcap_datalink(pkts)) == 0)
Expand All @@ -87,6 +88,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
pcap_close(pkts);
remove(pcap_path);
free(pcap_path);
ndpi_term_serializer(&workflow->ndpi_serializer);
return 0;
}

Expand All @@ -104,7 +106,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
ndpi_risk flow_risk;

memcpy(packet_checked, pkt, header->caplen);
ndpi_workflow_process_packet(workflow, header, packet_checked, &flow_risk, NULL);
ndpi_workflow_process_packet(workflow, header, packet_checked, &flow_risk);
free(packet_checked);
}

Expand Down
21 changes: 19 additions & 2 deletions fuzz/fuzz_process_packet.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
#include <stdio.h>

struct ndpi_detection_module_struct *ndpi_info_mod = NULL;
static ndpi_serializer json_serializer = {};
static ndpi_serializer csv_serializer = {};

int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
uint8_t protocol_was_guessed;
Expand All @@ -17,12 +19,27 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
ndpi_set_log_level(ndpi_info_mod, 4);
ndpi_set_debug_bitmask(ndpi_info_mod, debug_bitmask);
ndpi_finalize_initialization(ndpi_info_mod);
ndpi_init_serializer(&json_serializer, ndpi_serialization_format_json);
ndpi_init_serializer(&csv_serializer, ndpi_serialization_format_csv);
}

struct ndpi_flow_struct *ndpi_flow = ndpi_flow_malloc(SIZEOF_FLOW_STRUCT);
memset(ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, Data, Size, 0);
ndpi_detection_giveup(ndpi_info_mod, ndpi_flow, 1, &protocol_was_guessed);
ndpi_protocol detected_protocol =
ndpi_detection_process_packet(ndpi_info_mod, ndpi_flow, Data, Size, 0);
ndpi_protocol guessed_protocol =
ndpi_detection_giveup(ndpi_info_mod, ndpi_flow, 1, &protocol_was_guessed);

ndpi_reset_serializer(&json_serializer);
ndpi_reset_serializer(&csv_serializer);
if (protocol_was_guessed == 0)
{
ndpi_dpi2json(ndpi_info_mod, ndpi_flow, detected_protocol, &json_serializer);
ndpi_dpi2json(ndpi_info_mod, ndpi_flow, detected_protocol, &csv_serializer);
} else {
ndpi_dpi2json(ndpi_info_mod, ndpi_flow, guessed_protocol, &json_serializer);
ndpi_dpi2json(ndpi_info_mod, ndpi_flow, guessed_protocol, &csv_serializer);
}
ndpi_free_flow(ndpi_flow);

return 0;
Expand Down
Loading