diff --git a/CHANGES.md b/CHANGES.md index 5d765b5038b..5d9a1a072c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,9 +5,14 @@ - **irmin-pack** - Add `Irmin_pack_unix.Stats.Latest_gc` which is now the parameter of GC's `finished` callback (#2089, @Ngoguey42) + - Add `Gc.oldest_live_commit` which returns the key of the commit on which the + latest gc was called on. (#2110, @icristescu) ### Changed +- **irmin-pack** + - Upgraded on-disk format to version 4. (#2110, @icristescu) + ### Fixed ## 3.4.2 (2022-10-06) diff --git a/src/irmin-pack/unix/control_file.ml b/src/irmin-pack/unix/control_file.ml index 5145f4d1a81..3d4b3dd05c0 100644 --- a/src/irmin-pack/unix/control_file.ml +++ b/src/irmin-pack/unix/control_file.ml @@ -20,6 +20,12 @@ include Control_file_intf module Plv3 = struct include Payload_v3 + let of_bin_string = Irmin.Type.of_bin_string t |> Irmin.Type.unstage +end + +module Plv4 = struct + include Payload_v4 + let of_bin_string = Irmin.Type.of_bin_string t |> Irmin.Type.unstage let to_bin_string = Irmin.Type.to_bin_string t |> Irmin.Type.unstage end @@ -29,10 +35,11 @@ module Version = Irmin_pack.Version module Data (Io : Io.S) = struct (** Type of what's encoded in the control file. The variant tag is encoded as a [Version.t]. *) - type t = V3 of Plv3.t + type t = V3 of Plv3.t | V4 of Plv4.t let to_bin_string = function - | V3 payload -> Version.to_bin `V3 ^ Plv3.to_bin_string payload + | V3 _ -> assert false + | V4 payload -> Version.to_bin `V4 ^ Plv4.to_bin_string payload let of_bin_string s = let open Result_syntax in @@ -46,6 +53,7 @@ module Data (Io : Io.S) = struct | None -> Error (`Unknown_major_pack_version left) | Some `V3 when len > Io.page_size -> Error `Corrupted_control_file | Some `V3 -> Ok `V3 + | Some `V4 -> Ok `V4 | Some (`V1 | `V2) -> assert false in match version with @@ -53,6 +61,10 @@ module Data (Io : Io.S) = struct match Plv3.of_bin_string right with | Ok x -> Ok (V3 x) | Error _ -> Error `Corrupted_control_file) + | `V4 -> ( + match Plv4.of_bin_string right with + | Ok x -> Ok (V4 x) + | Error _ -> Error `Corrupted_control_file) end module Make (Io : Io.S) = struct @@ -61,12 +73,35 @@ module Make (Io : Io.S) = struct type t = { io : Io.t; mutable payload : Latest_payload.t } + let upgrade_v3_to_v4 (pl : Payload_v3.t) : Payload_v4.t = + let status = + match pl.status with + | From_v1_v2_post_upgrade x -> Payload_v4.From_v1_v2_post_upgrade x + | From_v3_no_gc_yet -> No_gc_yet + | From_v3_used_non_minimal_indexing_strategy -> + Used_non_minimal_indexing_strategy + | From_v3_gced x -> + Gced + { + suffix_start_offset = x.suffix_start_offset; + generation = x.generation; + oldest_live_commit_offset = x.suffix_start_offset; + } + | _ -> assert false + in + { + dict_end_poff = pl.dict_end_poff; + suffix_end_poff = pl.suffix_end_poff; + status; + upgraded_from_v3_to_v4 = true; + } + let write io payload = - let s = Data.(to_bin_string (V3 payload)) in + let s = Data.(to_bin_string (V4 payload)) in (* The data must fit inside a single page for atomic updates of the file. This is only true for some file systems. This system will have to be - reworked for [V3]. *) + reworked for [V4]. *) assert (String.length s <= Io.page_size); Io.write_string io ~off:Int63.zero s @@ -80,7 +115,10 @@ module Make (Io : Io.S) = struct If [string] is larger than a page, it either means that the file is corrupted or that the major version is not supported. Either way it will be handled by [Data.of_bin_string]. *) - Data.of_bin_string string + let+ payload = Data.of_bin_string string in + match payload with + | V4 payload -> payload + | V3 payload -> upgrade_v3_to_v4 payload let create_rw ~path ~overwrite payload = let open Result_syntax in @@ -91,8 +129,7 @@ module Make (Io : Io.S) = struct let open_ ~path ~readonly = let open Result_syntax in let* io = Io.open_ ~path ~readonly in - let+ data = read io in - let payload = match data with Data.V3 payload -> payload in + let+ payload = read io in { io; payload } let close t = Io.close t.io @@ -103,8 +140,7 @@ module Make (Io : Io.S) = struct let open Result_syntax in if not @@ Io.readonly t.io then Error `Rw_not_allowed else - let+ data = read t.io in - let payload = match data with Data.V3 payload -> payload in + let+ payload = read t.io in t.payload <- payload let set_payload t payload = diff --git a/src/irmin-pack/unix/control_file_intf.ml b/src/irmin-pack/unix/control_file_intf.ml index 643f3fdf877..7c107b7393b 100644 --- a/src/irmin-pack/unix/control_file_intf.ml +++ b/src/irmin-pack/unix/control_file_intf.ml @@ -103,7 +103,64 @@ module Payload_v3 = struct extensions *) end -module Latest_payload = Payload_v3 +module Payload_v4 = struct + type gced = { + suffix_start_offset : int63; + generation : int; + oldest_live_commit_offset : int63; + } + [@@deriving irmin] + (** Similar to [from_v3_gced]. [oldest_live_commit_offset] is the commit on + which the latest gc was called on. *) + + (** [From_v1_v2_post_upgrade] similar to [Payload_v3.From_v1_v2_post_upgrade] + + [No_gc_yet] corresponds to a pack store that was created using [`V3] or + above. It never underwent a GC. + + [Used_non_minimal_indexing_strategy] corresponds to a pack store that was + created using [`V3] or above. It never underwent a GC and it will never be + possible to GC it because entries were pushed using a non-minimal indexing + strategy. + + [Gced] is a [`V3] or [`V4] store that was GCed at least once. + + The [T*] tags are provisional tags that the binary decoder is aware of and + that may in the future be used to add features to the [`V3] payload. *) + type status = + | From_v1_v2_post_upgrade of Payload_v3.from_v1_v2_post_upgrade + | No_gc_yet + | Used_non_minimal_indexing_strategy + | Gced of gced + | T1 + | T2 + | T3 + | T4 + | T5 + | T6 + | T7 + | T8 + | T9 + | T10 + | T11 + | T12 + | T13 + | T14 + | T15 + [@@deriving irmin] + + type t = { + dict_end_poff : int63; + suffix_end_poff : int63; + status : status; + upgraded_from_v3_to_v4 : bool; + } + [@@deriving irmin] + (** Similar to [`V3] payload. [upgraded_from_v3_to_v4] recalls if the store + was originally created in [`V3]. *) +end + +module Latest_payload = Payload_v4 module type S = sig (** Abstraction for irmin-pack's control file. @@ -192,8 +249,9 @@ module type S = sig end module type Sigs = sig - module Latest_payload = Payload_v3 + module Latest_payload = Payload_v4 module Payload_v3 = Payload_v3 + module Payload_v4 = Payload_v4 module type S = S diff --git a/src/irmin-pack/unix/dispatcher.ml b/src/irmin-pack/unix/dispatcher.ml index 555bbf16648..70854fd41c2 100644 --- a/src/irmin-pack/unix/dispatcher.ml +++ b/src/irmin-pack/unix/dispatcher.ml @@ -58,13 +58,13 @@ module Make (Fm : File_manager.S with module Io = Io.Unix) : let suffix_start_offset t = let pl = Control.payload (Fm.control t.fm) in match pl.status with - | Payload.From_v1_v2_post_upgrade _ - | From_v3_used_non_minimal_indexing_strategy | From_v3_no_gc_yet -> + | Payload.From_v1_v2_post_upgrade _ | Used_non_minimal_indexing_strategy + | No_gc_yet -> Int63.zero | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> assert false - | From_v3_gced { suffix_start_offset; _ } -> suffix_start_offset + | Gced { suffix_start_offset; _ } -> suffix_start_offset (* The suffix only know the real offsets, it is in the dispatcher that global offsets are translated into real ones (i.e. in prefix or suffix offsets). *) diff --git a/src/irmin-pack/unix/ext.ml b/src/irmin-pack/unix/ext.ml index 3ffe9b5c3d1..b1910a1f3eb 100644 --- a/src/irmin-pack/unix/ext.ml +++ b/src/irmin-pack/unix/ext.ml @@ -250,14 +250,6 @@ module Maker (Config : Conf.S) = struct (Irmin.Type.to_string XKey.t commit_key)) | Some (k, _kind) -> Ok k) in - let offset = - let state : _ Pack_key.state = Pack_key.inspect commit_key in - match state with - | Direct x -> - let len = x.length |> Int63.of_int in - Int63.Syntax.(x.offset + len) - | Indexed _ -> assert false - in let root = Conf.root t.config in let* () = if not (File_manager.gc_allowed t.fm) then Error `Gc_disallowed @@ -266,7 +258,7 @@ module Maker (Config : Conf.S) = struct let current_generation = File_manager.generation t.fm in let next_generation = current_generation + 1 in let gc = - Gc.v ~root ~generation:next_generation ~unlink ~offset + Gc.v ~root ~generation:next_generation ~unlink ~dispatcher:t.dispatcher ~fm:t.fm ~contents:t.contents ~node:t.node ~commit:t.commit commit_key in @@ -318,6 +310,31 @@ module Maker (Config : Conf.S) = struct | Some { use_auto_finalisation = true; _ } -> let* _ = finalise_exn ~wait:false t in Lwt.return_unit + + let oldest_live_commit t = + let pl = Control.payload (File_manager.control t.fm) in + match pl.status with + | From_v1_v2_post_upgrade _ | Used_non_minimal_indexing_strategy + | No_gc_yet -> + None + | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 + | T14 | T15 -> + assert false + | Gced { oldest_live_commit_offset = offset; _ } -> ( + let entry = + Commit.CA.read_and_decode_entry_prefix ~off:offset + t.dispatcher + in + match Commit.CA.Entry_prefix.total_entry_length entry with + | None -> + (* Stores which support gc, also have a length in the header + of all their entries. *) + assert false + | Some length -> + let key = + Pack_key.v_direct ~offset ~length ~hash:entry.hash + in + Some key) end let batch t f = @@ -573,6 +590,7 @@ module Maker (Config : Conf.S) = struct let is_allowed repo = File_manager.gc_allowed repo.X.Repo.fm let cancel repo = X.Repo.Gc.cancel repo + let oldest_live_commit = X.Repo.Gc.oldest_live_commit end module Traverse_pack_file = Traverse_pack_file.Make (struct diff --git a/src/irmin-pack/unix/file_manager.ml b/src/irmin-pack/unix/file_manager.ml index 55875513a2f..3690bb5f3d0 100644 --- a/src/irmin-pack/unix/file_manager.ml +++ b/src/irmin-pack/unix/file_manager.ml @@ -77,14 +77,14 @@ struct t.suffix_consumers <- { after_flush } :: t.suffix_consumers let generation = function - | Payload.From_v1_v2_post_upgrade _ - | From_v3_used_non_minimal_indexing_strategy | From_v3_no_gc_yet -> + | Payload.From_v1_v2_post_upgrade _ | Used_non_minimal_indexing_strategy + | No_gc_yet -> 0 | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> (* Unreachable *) assert false - | From_v3_gced x -> x.generation + | Gced x -> x.generation (** Flush stages ************************************************************* @@ -130,17 +130,17 @@ struct let status = match pl.status with | From_v1_v2_post_upgrade _ -> pl.status - | From_v3_gced _ -> pl.status - | From_v3_no_gc_yet -> + | Gced _ -> pl.status + | No_gc_yet -> if Irmin_pack.Indexing_strategy.is_minimal t.indexing_strategy then pl.status else ( [%log.warn - "Updating the control file from [From_v3] to \ - [From_v3_used_non_minimal_indexing_strategy]. It won't be \ - possible to GC this irmin-pack store anymore."]; - Payload.From_v3_used_non_minimal_indexing_strategy) - | From_v3_used_non_minimal_indexing_strategy -> pl.status + "Updating the control file to \ + [Used_non_minimal_indexing_strategy]. It won't be possible \ + to GC this irmin-pack store anymore."]; + Payload.Used_non_minimal_indexing_strategy) + | Used_non_minimal_indexing_strategy -> pl.status | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> assert false @@ -294,10 +294,10 @@ struct let pl : Payload.t = Control.payload control in let generation = match pl.status with - | From_v1_v2_post_upgrade _ | From_v3_no_gc_yet - | From_v3_used_non_minimal_indexing_strategy -> + | From_v1_v2_post_upgrade _ | No_gc_yet + | Used_non_minimal_indexing_strategy -> 0 - | From_v3_gced x -> x.generation + | Gced x -> x.generation | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> assert false @@ -440,10 +440,15 @@ struct in let* control = let open Payload in - let status = From_v3_no_gc_yet in + let status = No_gc_yet in let pl = let z = Int63.zero in - { dict_end_poff = z; suffix_end_poff = z; status } + { + dict_end_poff = z; + suffix_end_poff = z; + status; + upgraded_from_v3_to_v4 = false; + } in create_control_file ~overwrite config pl in @@ -468,11 +473,11 @@ struct let* dead_header_size = match pl.status with | From_v1_v2_post_upgrade _ -> Ok legacy_io_header_size - | From_v3_gced _ -> + | Gced _ -> let indexing_strategy = Conf.indexing_strategy config in if Irmin_pack.Indexing_strategy.is_minimal indexing_strategy then Ok 0 else Error `Only_minimal_indexing_strategy_allowed - | From_v3_no_gc_yet | From_v3_used_non_minimal_indexing_strategy -> Ok 0 + | No_gc_yet | Used_non_minimal_indexing_strategy -> Ok 0 | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> Error `V3_store_from_the_future @@ -527,7 +532,14 @@ struct From_v1_v2_post_upgrade { entry_offset_at_upgrade_to_v3 = suffix_end_poff } in - let pl = { dict_end_poff; suffix_end_poff; status } in + let pl = + { + dict_end_poff; + suffix_end_poff; + status; + upgraded_from_v3_to_v4 = false; + } + in create_control_file ~overwrite:false config pl in let* () = Control.close control in @@ -580,9 +592,7 @@ struct let* dead_header_size = match pl.status with | From_v1_v2_post_upgrade _ -> Ok legacy_io_header_size - | From_v3_no_gc_yet | From_v3_gced _ - | From_v3_used_non_minimal_indexing_strategy -> - Ok 0 + | No_gc_yet | Gced _ | Used_non_minimal_indexing_strategy -> Ok 0 | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> Error `V3_store_from_the_future @@ -653,7 +663,8 @@ struct | `Unknown_major_pack_version _ ) as e -> e) - let swap t ~generation ~new_suffix_start_offset ~new_suffix_end_offset = + let swap t ~generation ~new_suffix_start_offset ~new_suffix_end_offset + ~oldest_live_commit_offset = let open Result_syntax in [%log.debug "Gc in main: swap %d %#d %#d" generation @@ -681,13 +692,14 @@ struct let status = match pl.status with | From_v1_v2_post_upgrade _ -> assert false - | From_v3_used_non_minimal_indexing_strategy -> assert false + | Used_non_minimal_indexing_strategy -> assert false | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> assert false - | From_v3_gced _ | From_v3_no_gc_yet -> + | Gced _ | No_gc_yet -> let suffix_start_offset = new_suffix_start_offset in - From_v3_gced { suffix_start_offset; generation } + Gced + { suffix_start_offset; generation; oldest_live_commit_offset } in { pl with status; suffix_end_poff } in @@ -705,10 +717,9 @@ struct let generation t = let pl = Control.payload t.control in match pl.status with - | From_v1_v2_post_upgrade _ | From_v3_used_non_minimal_indexing_strategy -> - 0 - | From_v3_no_gc_yet -> 0 - | From_v3_gced x -> x.generation + | From_v1_v2_post_upgrade _ | Used_non_minimal_indexing_strategy -> 0 + | No_gc_yet -> 0 + | Gced x -> x.generation | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> (* Unreachable *) @@ -717,9 +728,8 @@ struct let gc_allowed t = let pl = Control.payload t.control in match pl.status with - | From_v1_v2_post_upgrade _ | From_v3_used_non_minimal_indexing_strategy -> - false - | From_v3_no_gc_yet | From_v3_gced _ -> true + | From_v1_v2_post_upgrade _ | Used_non_minimal_indexing_strategy -> false + | No_gc_yet | Gced _ -> true | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> (* Unreachable *) diff --git a/src/irmin-pack/unix/file_manager_intf.ml b/src/irmin-pack/unix/file_manager_intf.ml index 070bb0a629d..cab9773ef6a 100644 --- a/src/irmin-pack/unix/file_manager_intf.ml +++ b/src/irmin-pack/unix/file_manager_intf.ml @@ -243,6 +243,7 @@ module type S = sig generation:int -> new_suffix_start_offset:int63 -> new_suffix_end_offset:int63 -> + oldest_live_commit_offset:int63 -> (unit, [> Errs.t ]) result (** Swaps to using files from the GC [generation]. The offsets [new_suffix_start_offset] and [new_suffix_end_offset] are used to properly diff --git a/src/irmin-pack/unix/gc.ml b/src/irmin-pack/unix/gc.ml index 7586b33d8a5..50e450a2316 100644 --- a/src/irmin-pack/unix/gc.ml +++ b/src/irmin-pack/unix/gc.ml @@ -39,10 +39,19 @@ module Make (Args : Gc_args.S) = struct commit : read Commit_store.t; mutable partial_stats : Gc_stats.Main.t; mutable resulting_stats : Stats.Latest_gc.stats option; + oldest_live_commit_offset : int63; } - let v ~root ~generation ~unlink ~offset ~dispatcher ~fm ~contents ~node - ~commit commit_key = + let v ~root ~generation ~unlink ~dispatcher ~fm ~contents ~node ~commit + commit_key = + let offset, oldest_live_commit_offset = + let state : _ Pack_key.state = Pack_key.inspect commit_key in + match state with + | Direct x -> + let len = x.length |> Int63.of_int in + (Int63.Syntax.(x.offset + len), x.offset) + | Indexed _ -> assert false + in let partial_stats = let commit_offset = offset in let before_suffix_start_offset = @@ -83,6 +92,7 @@ module Make (Args : Gc_args.S) = struct let partial_stats = Gc_stats.Main.finish_current_step partial_stats "before finalise" in + { root; generation; @@ -98,6 +108,7 @@ module Make (Args : Gc_args.S) = struct commit; partial_stats; resulting_stats = None; + oldest_live_commit_offset; } let open_new_suffix ~end_poff { root; generation; _ } = @@ -142,6 +153,7 @@ module Make (Args : Gc_args.S) = struct let* () = Fm.swap t.fm ~generation:t.generation ~new_suffix_start_offset ~new_suffix_end_offset + ~oldest_live_commit_offset:t.oldest_live_commit_offset in (* No need to purge dict here, as it is global to the store. *) diff --git a/src/irmin-pack/unix/gc.mli b/src/irmin-pack/unix/gc.mli index ca8ac3a5082..cd5836f6fe1 100644 --- a/src/irmin-pack/unix/gc.mli +++ b/src/irmin-pack/unix/gc.mli @@ -27,7 +27,6 @@ module Make (Args : Gc_args.S) : sig root:string -> generation:int -> unlink:bool -> - offset:int63 -> dispatcher:Args.Dispatcher.t -> fm:Args.Fm.t -> contents:read Args.Contents_store.t -> diff --git a/src/irmin-pack/unix/s.ml b/src/irmin-pack/unix/s.ml index 5f893011702..78fd68337b7 100644 --- a/src/irmin-pack/unix/s.ml +++ b/src/irmin-pack/unix/s.ml @@ -127,6 +127,10 @@ module type S = sig val is_allowed : repo -> bool (** [is_allowed repo] returns true if a gc can be run on the store. *) + + val oldest_live_commit : repo -> commit_key option + (** [oldest_live_commit repo] returns the commit key on which the latest, + finished gc was called on. *) end val integrity_check_inodes : diff --git a/src/irmin-pack/version.ml b/src/irmin-pack/version.ml index fb296b93500..e355bc90da5 100644 --- a/src/irmin-pack/version.ml +++ b/src/irmin-pack/version.ml @@ -17,13 +17,22 @@ (* For every new version, update the [version] type and [versions] headers. *) -type t = [ `V1 | `V2 | `V3 ] [@@deriving irmin] +type t = [ `V1 | `V2 | `V3 | `V4 ] [@@deriving irmin] let latest = `V3 -let enum = [ (`V1, "00000001"); (`V2, "00000002"); (`V3, "00000003") ] -let pp = Fmt.of_to_string (function `V1 -> "v1" | `V2 -> "v2" | `V3 -> "v3") + +let enum = + [ (`V1, "00000001"); (`V2, "00000002"); (`V3, "00000003"); (`V4, "00000004") ] + +let pp = + Fmt.of_to_string (function + | `V1 -> "v1" + | `V2 -> "v2" + | `V3 -> "v3" + | `V4 -> "v4") + let to_bin v = List.assoc v enum -let to_int = function `V1 -> 1 | `V2 -> 2 | `V3 -> 3 +let to_int = function `V1 -> 1 | `V2 -> 2 | `V3 -> 3 | `V4 -> 4 let compare a b = Int.compare (to_int a) (to_int b) let encode_bin t f = to_bin t |> f @@ -34,6 +43,7 @@ let decode_bin s offref = | "00000001" -> `V1 | "00000002" -> `V2 | "00000003" -> `V3 + | "00000004" -> `V4 | _ -> failwith "Couldn't decode pack version" in offref := !offref + 8; diff --git a/src/irmin-pack/version.mli b/src/irmin-pack/version.mli index 5480eb7b281..0169db76e30 100644 --- a/src/irmin-pack/version.mli +++ b/src/irmin-pack/version.mli @@ -26,7 +26,7 @@ file). The upgrade of a store to [`V3] was done silently when opening a store in rw mode. *) -type t = [ `V1 | `V2 | `V3 ] [@@deriving irmin] +type t = [ `V1 | `V2 | `V3 | `V4 ] [@@deriving irmin] (** The type for version numbers. *) val compare : t -> t -> int diff --git a/src/irmin-tezos-utils/show.ml b/src/irmin-tezos-utils/show.ml index aa977cd0d00..2678d150e6e 100644 --- a/src/irmin-tezos-utils/show.ml +++ b/src/irmin-tezos-utils/show.ml @@ -720,10 +720,10 @@ let main store_path info_last_path info_next_path index_path = let max_offset = (* TODO: Rename [pl.suffix_end_poff] to [suffix_length] *) match pl.status with - | From_v1_v2_post_upgrade _ | From_v3_no_gc_yet - | From_v3_used_non_minimal_indexing_strategy -> + | From_v1_v2_post_upgrade _ | No_gc_yet | Used_non_minimal_indexing_strategy + -> pl.suffix_end_poff - | From_v3_gced x -> Int63.add x.suffix_start_offset pl.suffix_end_poff + | Gced x -> Int63.add x.suffix_start_offset pl.suffix_end_poff | T1 | T2 | T3 | T4 | T5 | T6 | T7 | T8 | T9 | T10 | T11 | T12 | T13 | T14 | T15 -> assert false diff --git a/test/irmin-cli/test_command_line.t b/test/irmin-cli/test_command_line.t index a65906508e9..14f316d0b52 100644 --- a/test/irmin-cli/test_command_line.t +++ b/test/irmin-cli/test_command_line.t @@ -6,11 +6,11 @@ Create default config file Set a/b/c => 123 in ./test $ irmin set a/b/c 123 - irmin: [WARNING] Updating the control file from [From_v3] to [From_v3_used_non_minimal_indexing_strategy]. It won't be possible to GC this irmin-pack store anymore. + irmin: [WARNING] Updating the control file to [Used_non_minimal_indexing_strategy]. It won't be possible to GC this irmin-pack store anymore. Set foo => bar in ./test1 $ irmin set --root ./test1 foo bar - irmin: [WARNING] Updating the control file from [From_v3] to [From_v3_used_non_minimal_indexing_strategy]. It won't be possible to GC this irmin-pack store anymore. + irmin: [WARNING] Updating the control file to [Used_non_minimal_indexing_strategy]. It won't be possible to GC this irmin-pack store anymore. Check for a/b/c in ./test $ irmin get --root ./test a/b/c @@ -91,6 +91,6 @@ Check mismatched hash function Clone a local repo $ irmin clone --root ./cloned ./test - irmin: [WARNING] Updating the control file from [From_v3] to [From_v3_used_non_minimal_indexing_strategy]. It won't be possible to GC this irmin-pack store anymore. + irmin: [WARNING] Updating the control file to [Used_non_minimal_indexing_strategy]. It won't be possible to GC this irmin-pack store anymore. $ irmin get --root ./cloned a/b/c 123 diff --git a/test/irmin-pack/data/version_3_minimal_gced/index/log b/test/irmin-pack/data/version_3_minimal_gced/index/log new file mode 100644 index 00000000000..e16fefc09ae Binary files /dev/null and b/test/irmin-pack/data/version_3_minimal_gced/index/log differ diff --git a/test/irmin-pack/data/version_3_minimal_gced/store.1.mapping b/test/irmin-pack/data/version_3_minimal_gced/store.1.mapping new file mode 100644 index 00000000000..3d997fc6f96 Binary files /dev/null and b/test/irmin-pack/data/version_3_minimal_gced/store.1.mapping differ diff --git a/test/irmin-pack/data/version_3_minimal_gced/store.1.prefix b/test/irmin-pack/data/version_3_minimal_gced/store.1.prefix new file mode 100644 index 00000000000..6bfd270c6e5 Binary files /dev/null and b/test/irmin-pack/data/version_3_minimal_gced/store.1.prefix differ diff --git a/test/irmin-pack/data/version_3_minimal_gced/store.1.suffix b/test/irmin-pack/data/version_3_minimal_gced/store.1.suffix new file mode 100644 index 00000000000..7fa4cb187a7 Binary files /dev/null and b/test/irmin-pack/data/version_3_minimal_gced/store.1.suffix differ diff --git a/test/irmin-pack/data/version_3_minimal_gced/store.branches b/test/irmin-pack/data/version_3_minimal_gced/store.branches new file mode 100644 index 00000000000..b35ec9eb860 Binary files /dev/null and b/test/irmin-pack/data/version_3_minimal_gced/store.branches differ diff --git a/test/irmin-pack/data/version_3_minimal_gced/store.control b/test/irmin-pack/data/version_3_minimal_gced/store.control new file mode 100644 index 00000000000..14b85bca92d Binary files /dev/null and b/test/irmin-pack/data/version_3_minimal_gced/store.control differ diff --git a/test/irmin-pack/data/version_3_minimal_gced/store.dict b/test/irmin-pack/data/version_3_minimal_gced/store.dict new file mode 100644 index 00000000000..167a469b4f0 Binary files /dev/null and b/test/irmin-pack/data/version_3_minimal_gced/store.dict differ diff --git a/test/irmin-pack/test_gc.ml b/test/irmin-pack/test_gc.ml index 34122245086..c89f087f502 100644 --- a/test/irmin-pack/test_gc.ml +++ b/test/irmin-pack/test_gc.ml @@ -567,6 +567,33 @@ module Gc = struct let* () = check_1 t c1_again in S.Repo.close t.repo + (** Check [Gc.oldest_live_commit]. *) + let oldest_live_commit () = + let* t = init () in + let check_oldest_live_commit expected = + let got = S.Gc.oldest_live_commit t.repo in + match (got, expected) with + | None, None -> () + | Some got, Some expected -> + Alcotest.check_repr S.commit_key_t "oldest_live_commit" got + (S.Commit.key expected) + | _ -> Alcotest.fail "Check of oldest_live_commit failed" + in + + let* t, c1 = commit_1 t in + let* t = checkout_exn t c1 in + check_oldest_live_commit None; + let* t, c2 = commit_2 t in + let* () = start_gc t c2 in + let* () = finalise_gc t in + check_oldest_live_commit (Some c2); + let* t = checkout_exn t c2 in + let* t, c3 = commit_3 t in + let* () = start_gc t c3 in + let* () = finalise_gc t in + check_oldest_live_commit (Some c3); + S.Repo.close t.repo + let tests = [ tc "Test one gc" one_gc; @@ -583,6 +610,7 @@ module Gc = struct tc "Test gc during batch" gc_during_batch; tc "Test add back gced commit" add_back_gced_commit; tc "Test gc on similar commits" gc_similar_commits; + tc "Test oldest live commit" oldest_live_commit; ] end diff --git a/test/irmin-pack/test_pack_version_bump.ml b/test/irmin-pack/test_pack_version_bump.ml index 66642889efc..17504529883 100644 --- a/test/irmin-pack/test_pack_version_bump.ml +++ b/test/irmin-pack/test_pack_version_bump.ml @@ -95,7 +95,7 @@ module Util = struct (** Get the version of the underlying file; file is assumed to exist; file is assumed to be an Irmin_pack.IO.Unix file *) - let io_get_version ~root : [ `V1 | `V2 | `V3 ] = + let io_get_version ~root : [ `V1 | `V2 | `V3 | `V4 ] = File_manager.version ~root |> Errs.raise_if_error let alco_check_version ~pos ~expected ~actual = diff --git a/test/irmin-pack/test_upgrade.ml b/test/irmin-pack/test_upgrade.ml index 07b11a98752..f7faf0d63d5 100644 --- a/test/irmin-pack/test_upgrade.ml +++ b/test/irmin-pack/test_upgrade.ml @@ -22,6 +22,10 @@ let archive_v2_minimal = "test" / "irmin-pack" / "data" / "version_2_minimal" let archive_v2_always = "test" / "irmin-pack" / "data" / "version_2_always" let archive_v3_minimal = "test" / "irmin-pack" / "data" / "version_3_minimal" let archive_v3_always = "test" / "irmin-pack" / "data" / "version_3_always" + +let archive_v3_minimal_gced = + "test" / "irmin-pack" / "data" / "version_3_minimal_gced" + let root_local_build = "_build" / "test-upgrade" type pack_entry = { @@ -86,7 +90,7 @@ let key_of_entry x = type start_mode = From_v2 | From_v3 | From_scratch [@@deriving irmin] type setup = { - indexing_strategy : [ `always | `minimal ]; + indexing_strategy : [ `always | `minimal | `minimal_gc ]; start_mode : start_mode; lru_size : int; } @@ -121,7 +125,8 @@ module Model = struct Hashtbl.replace t.dict "step-n01" () let preload_suffix t = - Hashtbl.replace t.suffix borphan.o (); + if t.setup.indexing_strategy <> `minimal_gc then + Hashtbl.replace t.suffix borphan.o (); Hashtbl.replace t.suffix b01.o (); Hashtbl.replace t.suffix n01.o (); Hashtbl.replace t.suffix n0.o (); @@ -459,6 +464,8 @@ let create_test_env setup = | From_v3 -> let root_archive = if setup.indexing_strategy = `always then archive_v3_always + else if setup.indexing_strategy = `minimal_gc then + archive_v3_minimal_gced else archive_v3_minimal in setup_test_env ~root_archive ~root_local_build @@ -523,7 +530,9 @@ let gc_rw t = Store.gc repo) in raise Skip_the_rest_of_that_test - | (From_v3 | From_scratch), `minimal -> Store.gc repo + | (From_v3 | From_scratch), `minimal + | (From_v3 | From_scratch), `minimal_gc -> + Store.gc repo in Lwt.return_unit @@ -667,6 +676,10 @@ let test start_mode indexing_strategy = let* () = test start_mode indexing_strategy 100 in Lwt.return_unit +let test_gced_store () = + let* () = test From_v3 `minimal_gc in + Lwt.return_unit + (** Product on indexing_strategy *) let test start_mode () = let* () = test start_mode `minimal in @@ -682,4 +695,6 @@ let tests = test From_v2 ()); Alcotest_lwt.test_case "upgrade From_scratch" `Quick (fun _switch () -> test From_scratch ()); + Alcotest_lwt.test_case "upgrade From_v3 after Gc" `Quick (fun _switch () -> + test_gced_store ()); ]