ndpiReader: allow to configure LRU caches TTL and size

example/ndpiReader.c

@@ -133,6 +133,9 @@ int enable_malloc_bins = 0;
 int max_malloc_bins = 14;
 int malloc_size_stats = 0;
 
+static int lru_cache_sizes[NDPI_LRUCACHE_MAX];
+static int lru_cache_ttls[NDPI_LRUCACHE_MAX];
+
 struct flow_info {
   struct ndpi_flow_info *flow;
   u_int16_t thread_id;
@@ -511,11 +514,15 @@ static void help(u_int long_help) {
          "  -A                        | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n"
          "  -M                        | Memory allocation stats on data-path (only by the library). It works only on single-thread configuration\n"
          "  -Z proto:value            | Set this value of aggressiveness for this protocol (0 to disable it). This flag can be used multiple times\n"
+         "  --lru-cache-size=NAME:size    | Specify the size for this LRU cache (0 to disable it). This flag can be used multiple times\n"
+         "  --lru-cache-ttl=NAME:size     | Specify the TTL [in seconds] for this LRU cache (0 to disable it). This flag can be used multiple times\n"
          ,
          human_readeable_string_len,
          min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection,
          max_num_reported_top_payloads, max_num_tcp_dissected_pkts, max_num_udp_dissected_pkts);
 
+  printf("\nLRU Cache names: ookla, bittorrent, zoom, stun, tls_cert, mining, msteams, stun_zoom\n");
+
 #ifndef WIN32
   printf("\nExcap (wireshark) options:\n"
          "  --extcap-interfaces\n"
@@ -559,6 +566,9 @@ static void help(u_int long_help) {
 }
 
 
+#define OPTLONG_VALUE_LRU_CACHE_SIZE	1000
+#define OPTLONG_VALUE_LRU_CACHE_TTL	1001
+
 static struct option longopts[] = {
   /* mandatory extcap options */
   { "extcap-interfaces", no_argument, NULL, '0'},
@@ -599,6 +609,9 @@ static struct option longopts[] = {
   { "result-path", required_argument, NULL, 'w'},
   { "quiet", no_argument, NULL, 'q'},
 
+  { "lru-cache-size", required_argument, NULL, OPTLONG_VALUE_LRU_CACHE_SIZE},
+  { "lru-cache-ttl", required_argument, NULL, OPTLONG_VALUE_LRU_CACHE_TTL},
+
   {0, 0, 0, 0}
 };
 
@@ -788,6 +801,52 @@ void printCSVHeader() {
   fprintf(csv_fp, "\n");
 }
 
+static int cache_idx_from_name(const char *name)
+{
+  if(strcmp(name, "ookla") == 0)
+    return NDPI_LRUCACHE_OOKLA;
+  if(strcmp(name, "bittorrent") == 0)
+    return NDPI_LRUCACHE_BITTORRENT;
+  if(strcmp(name, "zoom") == 0)
+    return NDPI_LRUCACHE_ZOOM;
+  if(strcmp(name, "stun") == 0)
+    return NDPI_LRUCACHE_STUN;
+  if(strcmp(name, "tls_cert") == 0)
+    return NDPI_LRUCACHE_TLS_CERT;
+  if(strcmp(name, "mining") == 0)
+    return NDPI_LRUCACHE_MINING;
+  if(strcmp(name, "msteams") == 0)
+    return NDPI_LRUCACHE_MSTEAMS;
+  if(strcmp(name, "stun_zoom") == 0)
+    return NDPI_LRUCACHE_STUN_ZOOM;
+  return -1;
+}
+
+static int parse_cache_param(char *param, int *cache_idx, int *param_value)
+{
+  char *saveptr, *tmp_str, *cache_str, *param_str;
+  int idx;
+
+  tmp_str = ndpi_strdup(param);
+  if(tmp_str) {
+    cache_str = strtok_r(tmp_str, ":", &saveptr);
+    if(cache_str) {
+      param_str = strtok_r(NULL, ":", &saveptr);
+      if(param_str) {
+        idx = cache_idx_from_name(cache_str);
+        if(idx >= 0) {
+          *cache_idx = idx;
+          *param_value = atoi(param_str);
+          ndpi_free(tmp_str);
+	  return 0;
+	}
+      }
+    }
+  }
+  ndpi_free(tmp_str);
+  return -1;
+}
+
 /* ********************************** */
 
 /**
@@ -804,6 +863,7 @@ static void parseOptions(int argc, char **argv) {
   u_int num_cores = sysconf(_SC_NPROCESSORS_ONLN);
 #endif
 #endif
+  int cache_idx, cache_size, cache_ttl;
 
 #ifdef USE_DPDK
   {
@@ -819,6 +879,11 @@ static void parseOptions(int argc, char **argv) {
   for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++)
     aggressiveness[i] = -1; /* Use the default value */
 
+  for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
+    lru_cache_sizes[i] = -1; /* Use the default value */
+    lru_cache_ttls[i] = -1; /* Use the default value */
+  }
+
   while((opt = getopt_long(argc, argv, "a:Ab:B:e:Ec:C:dDFf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zZ:q0123:456:7:89:m:MT:U:",
                            longopts, &option_idx)) != EOF) {
 #ifdef DEBUG_TRACE
@@ -1112,6 +1177,22 @@ static void parseOptions(int argc, char **argv) {
       init_prefs |= ndpi_enable_ja3_plus;
       break;
 
+    case OPTLONG_VALUE_LRU_CACHE_SIZE:
+      if(parse_cache_param(optarg, &cache_idx, &cache_size) == -1) {
+        printf("Invalid parameter [%s]\n", optarg);
+        exit(1);
+      }
+      lru_cache_sizes[cache_idx] = cache_size;
+      break;
+
+    case OPTLONG_VALUE_LRU_CACHE_TTL:
+      if(parse_cache_param(optarg, &cache_idx, &cache_ttl) == -1) {
+        printf("Invalid parameter [%s]\n", optarg);
+        exit(1);
+      }
+      lru_cache_ttls[cache_idx] = cache_ttl;
+      break;
+
     default:
 #ifdef DEBUG_TRACE
       if(trace) fprintf(trace, " #### Unknown option -%c: skipping it #### \n", opt);
@@ -2504,9 +2585,18 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
     ndpi_load_malicious_sha1_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _maliciousSHA1Path);
 
   /* Enable/disable/configure LRU caches size here */
-  ndpi_set_lru_cache_size(ndpi_thread_info[thread_id].workflow->ndpi_struct,
-			  NDPI_LRUCACHE_BITTORRENT, 32768);
+  for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
+    if(lru_cache_sizes[i] != -1)
+      ndpi_set_lru_cache_size(ndpi_thread_info[thread_id].workflow->ndpi_struct,
+			      i, lru_cache_sizes[i]);
+  }
+
   /* Enable/disable LRU caches TTL here */
+  for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
+    if(lru_cache_ttls[i] != -1)
+      ndpi_set_lru_cache_ttl(ndpi_thread_info[thread_id].workflow->ndpi_struct,
+			     i, lru_cache_ttls[i]);
+  }
 
   /* Set aggressiviness here */
   for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {

tests/cfgs/caches_cfg/config.txt

@@ -0,0 +1 @@
+--lru-cache-size=ookla:0 --lru-cache-ttl=msteams:1

tests/cfgs/caches_cfg/pcap/ookla.pcap

@@ -0,0 +1 @@
+../../default/pcap/ookla.pcap

tests/cfgs/caches_cfg/pcap/teams.pcap

@@ -0,0 +1 @@
+../../default/pcap/teams.pcap

tests/cfgs/caches_cfg/result/ookla.pcap.out

@@ -0,0 +1,38 @@
+Guessed flow protos:	1
+
+DPI Packets (TCP):	40	(6.67 pkts/flow)
+Confidence Match by port    : 1 (flows)
+Confidence DPI              : 5 (flows)
+Num dissector calls: 489 (81.50 diss/flow)
+LRU cache ookla:      0/0/0 (insert/search/found)
+LRU cache bittorrent: 0/3/0 (insert/search/found)
+LRU cache zoom:       0/0/0 (insert/search/found)
+LRU cache stun:       0/0/0 (insert/search/found)
+LRU cache tls_cert:   0/2/0 (insert/search/found)
+LRU cache mining:     0/1/0 (insert/search/found)
+LRU cache msteams:    0/0/0 (insert/search/found)
+LRU cache stun_zoom:  0/0/0 (insert/search/found)
+Automa host:          3/2 (search/found)
+Automa domain:        3/0 (search/found)
+Automa tls cert:      0/0 (search/found)
+Automa risk mask:     2/0 (search/found)
+Automa common alpns:  4/4 (search/found)
+Patricia risk mask:   12/0 (search/found)
+Patricia risk:        0/0 (search/found)
+Patricia protocols:   11/1 (search/found)
+
+TLS	29	23166	1
+HTTP_Proxy	10	2375	1
+Ookla	74	12870	4
+
+JA3 Host Stats: 
+		 IP Address                  	 # JA3C     
+	1	 192.168.1.128            	 2      
+
+
+	1	TCP 192.168.1.128:35830 <-> 89.96.108.170:8080 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 6][cat: Web/5][21 pkts/21216 bytes <-> 8 pkts/1950 bytes][Goodput ratio: 93/72][0.32 sec][Hostname/SNI: spd-pub-mi-01-01.fastwebnet.it][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: 0.832 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/61 274/280 62/109][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1010/244 1514/387 612/138][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][Risk Info: Expected on port 443][TLSv1.3][JA3C: c279b0189edb9269da7bc43dea5e0c36][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,4,0,0,0,0,4,9,0,9,0,0,0,0,0,4,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0]
+	2	TCP 192.168.1.128:48854 <-> 104.16.209.12:443 [proto: 91.191/TLS.Ookla][IP: 220/Cloudflare][Encrypted][Confidence: DPI][DPI packets: 6][cat: Network/14][8 pkts/1620 bytes <-> 6 pkts/3818 bytes][Goodput ratio: 67/89][0.06 sec][Hostname/SNI: www.speedtest.net][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.404 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/5 18/15 7/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 202/636 583/1514 181/646][TLSv1.3][JA3C: 579ccef312d18482fc42e2b822ca2430][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Firefox][Cipher: TLS_AES_128_GCM_SHA256][PLAIN TEXT (@oTAgOeedtest.net)][Plen Bins: 0,0,14,0,0,14,0,0,0,0,14,0,0,0,0,0,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,0,0]
+	3	TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][Hostname/SNI: massarosa-1.speedtest.welcomeitalia.it][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][URL: massarosa-1.speedtest.welcomeitalia.it/crossdomain.xml][StatusCode: 200][Content-Type: application/xml][Server: Apache/2.2.22 (Ubuntu)][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8][Risk: ** HTTP Obsolete Server **][Risk Score: 50][Risk Info: Obsolete Apache server 2.2.22][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	4	TCP 192.168.1.192:51156 <-> 89.96.108.170:8080 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 10][cat: Web/5][6 pkts/591 bytes <-> 4 pkts/1784 bytes][Goodput ratio: 32/85][0.05 sec][bytes ratio: -0.502 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/10 15/20 6/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/446 143/1514 31/617][PLAIN TEXT (gKRZvA)][Plen Bins: 0,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0]
+	5	TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	6	TCP 192.168.1.192:37790 <-> 185.157.229.246:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][6 pkts/454 bytes <-> 4 pkts/317 bytes][Goodput ratio: 11/14][0.06 sec][bytes ratio: 0.178 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/5 46/9 17/4][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 76/79 106/108 14/17][PLAIN TEXT (HELLO 2.9 )][Plen Bins: 50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]