replace simple IP pool free table with LRU implementation

apps/ergw_core/src/ergw_local_pool.erl

@@ -22,7 +22,7 @@
 	 terminate/2, code_change/3]).
 
 -record(state, {name, pools, ipv4_opts, ipv6_opts}).
--record(pool,  {name, type, id, base, size, first, last, shift, used, free, used_pool, free_pool}).
+-record(pool,  {name, type, id, base, size, first, last, shift, used, free, used_pool, free_lru}).
 -record(lease, {ip, client_id}).
 
 -include_lib("kernel/include/logger.hrl").
@@ -193,10 +193,10 @@ init_pools(Name, Ranges) ->
 	      Pools
       end, #{}, Ranges).
 
-init_table(Tid, Start, End) ->
+init_free_lru(Start, End) ->
     Length = End - Start + 1,
     IPsList = uint32_fy_shuffle:shuffle(Length),
-    true = ets:insert(Tid, IPsList).
+    lru:from_list(IPsList).
 
 handle_call({get, ClientId, Type, PrefixLen, ReqOpts}, _From, #state{pools = Pools} = State)
   when is_atom(Type), is_map_key({Type, PrefixLen}, Pools) ->
@@ -276,14 +276,13 @@ alloc_reply({error, _} = Error, _, _) ->
 
 init_pool(Name, Type, First, Last, Shift) ->
     UsedTid = ets:new(used_pool, [set, {keypos, #lease.ip}]),
-    FreeTid = ets:new(free_pool, [ordered_set]),
 
     Id = inet:ntoa(int2ip(Type, First)),
     Start = First bsr Shift,
     End = Last bsr Shift,
     Size = End - Start + 1,
     ?LOG(debug, "init Pool ~w ~p - ~p (~p)", [Id, Start, End, Size]),
-    init_table(FreeTid, Start, End),
+    FreeLRU = init_free_lru(Start, End),
 
     prometheus_gauge:declare([{name, ergw_local_pool_free},
 			      {labels, [name, type, id]},
@@ -297,20 +296,19 @@ init_pool(Name, Type, First, Last, Shift) ->
 	      base = Start, size = Size,
 	      first = First, last = Last, shift = Shift,
 	      used = 0, free = Size,
-	      used_pool = UsedTid, free_pool = FreeTid},
+	      used_pool = UsedTid, free_lru = FreeLRU},
     metrics_sync_gauges(Pool),
     ?LOG(debug, "init Pool state: ~p", [Pool]),
     Pool.
 
 allocate_ip(ClientId, ReqOpts,
 	    #pool{base = Base,
 		  used = Used, free = Free,
-		  used_pool = UsedTid, free_pool = FreeTid} = Pool0)
+		  used_pool = UsedTid, free_lru = FreeLRU} = Pool0)
   when Free =/= 0 ->
     ?LOG(debug, "~w: Allocate Pool: ~p", [self(), Pool0]),
 
-    {_, Id} = Key = ets:first(FreeTid),
-    ets:delete(FreeTid, Key),
+    {ok, Id} = lru:pop(FreeLRU),
     ets:insert(UsedTid, #lease{ip = Id, client_id = ClientId}),
     IP = id2ip(Base + Id, ReqOpts, Pool0),
     Pool = Pool0#pool{used = Used + 1, free = Free - 1},
@@ -322,14 +320,13 @@ allocate_ip(_ClientId, _ReqOpts, Pool) ->
 release_ip(IP, #pool{base = Base, first = First, last = Last,
 		     shift = Shift,
 		     used = Used, free = Free,
-		     used_pool = UsedTid, free_pool = FreeTid} = Pool0)
+		     used_pool = UsedTid, free_lru = FreeLRU} = Pool0)
   when IP >= First andalso IP =< Last ->
     Id = (IP bsr Shift) - Base,
 
     case ets:take(UsedTid, Id) of
 	[_] ->
-	    Now = erlang:monotonic_time(),
-	    ets:insert(FreeTid, {{Now, Id}}),
+	    lru:push(Id, FreeLRU),
 	    Pool = Pool0#pool{used = Used - 1, free = Free + 1},
 	    metrics_sync_gauges(Pool),
 	    Pool;
@@ -348,13 +345,12 @@ take_ip(ClientId, IP, ReqOpts,
 	#pool{base = Base, first = First, last = Last,
 	      shift = Shift,
 	      used = Used, free = Free,
-	      used_pool = UsedTid, free_pool = FreeTid} = Pool0)
+	      used_pool = UsedTid, free_lru = FreeLRU} = Pool0)
   when IP >= First andalso IP =< Last ->
     Id = (IP bsr Shift) - Base,
 
-    case ets:match_object(FreeTid, {{'_', Id}}) of
-	[{Key}] ->
-	    ets:delete(FreeTid, Key),
+    case lru:take(Id, FreeLRU) of
+	ok ->
 	    ets:insert(UsedTid, #lease{ip = Id, client_id = ClientId}),
 	    Pool = Pool0#pool{used = Used + 1, free = Free - 1},
 	    metrics_sync_gauges(Pool),

apps/ergw_core/src/lru.erl

@@ -0,0 +1,66 @@
+%% Copyright 2021, Travelping GmbH <[email protected]>
+
+%% This program is free software; you can redistribute it and/or
+%% modify it under the terms of the GNU General Public License
+%% as published by the Free Software Foundation; either version
+%% 2 of the License, or (at your option) any later version.
+
+%% ets LRU cache for IP pools, scales to at least 2^24 entries.
+
+-module(lru).
+
+-record(lru, {queue, index}).
+
+-export([from_list/1, to_list/1,
+	 pop/1, push/2, take/2,
+	 valid/1, info/1]).
+
+-ignore_xref([?MODULE]).
+
+from_list([{{_, _}}|_] = List) ->
+    Queue = ets:new(entries, [ordered_set]),
+    Index = ets:new(index, [set]),
+
+    IndexList = [{Key, QEntry} || {{_, Key} = QEntry} <- List],
+
+    true = ets:insert(Queue, List),
+    true = ets:insert(Index, IndexList),
+    #lru{queue = Queue, index = Index}.
+
+to_list(#lru{queue = Queue}) ->
+    ets:tab2list(Queue).
+
+pop(#lru{queue = Queue, index = Index}) ->
+    case ets:first(Queue) of
+	{_, Key} = Entry ->
+	    ets:delete(Queue, Entry),
+	    ets:delete(Index, Key),
+	    {ok, Key};
+	'$end_of_table' ->
+	    {error, empty}
+    end.
+
+push(Key, #lru{queue = Queue, index = Index}) ->
+    Now = erlang:monotonic_time(),
+    case ets:insert_new(Index, {Key, Now}) of
+	true ->
+	    ets:insert(Queue, {{Now, Key}}),
+	    ok;
+	false ->
+	    erlang:error(badarg, [Key])
+    end.
+
+take(Key, #lru{queue = Queue, index = Index}) ->
+    case ets:take(Index, Key) of
+	[{_, QEntry}] ->
+	    ets:delete(Queue, QEntry),
+	    ok;
+	_ ->
+	    {error, not_found}
+    end.
+
+valid(#lru{queue = Queue, index = Index}) ->
+    ets:info(Queue, size) =:= ets:info(Index, size).
+
+info(#lru{queue = Queue, index = Index}) ->
+    {ets:info(Queue, size), ets:info(Index, size)}.