Merge branch 'main' into toplevel_printers

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Upgrade to OCamlformat 0.26.0
+faab08a7dd9c275111cae51651a78fd62ac3e031

.ocamlformat

@@ -1,4 +1,4 @@
-version = 0.24.1
+version = 0.26.0
 profile = conventional
 
 ocaml-version = 4.08.0

CHANGES.md

@@ -2,33 +2,74 @@
 
 ### Added
 
+- **irmin-server**
+  - Added `irmin-server` package (#2031, @zshipko)
+- **irmin-client**
+  - Added `irmin-client` package to connect to `irmin-server` instances (#2031,
+    @zshipko)
 - **irmin**
-  - Change behavior of `Irmin.Conf.key` to disallow duplicate key names by
-    default. Add `allow_duplicate` optional argument to override. (@metanivek,
-    #2252)
   - Add pretty printers for `Commit`, `Tree`, `Info`, `Status`, `Branch` when
     using `utop` (@metanivek, #1839)
+
+### Fixed
+
+- **irmin-pack**
+  - Fix index integrity check for v3 stores (#2267, @metanivek)
+
+### Removed
+
+- **irmin-http**
+  - Removed `irmin-http` since it is not compatible with generic keys.
+    `irmin-grapqhl` or `irmin-server` should be used instead. (#1902, @zshipko)
+
+## 3.8.0 (2023-07-06)
+
+### Added
+
+- **irmin**
+  - Change behavior of `Irmin.Conf.key` to disallow duplicate key names by
+    default. Add `allow_duplicate` optional argument to override. (#2252,
+    @metanivek)
 
+- **irmin-pack**
+  - Add maximum memory as an alternative configuration option, `lru_max_memory`,
+    for setting LRU capacity. (@metanivek, #2254)
+
 ### Changed
 
 - **irmin**
-  - Lower bounds for `mtime` is now `2.0.0` (@patricoferris, #2166)
+  - Lower bound for `mtime` is now `2.0.0` (#2166, @patricoferris)
+
+- **irmin-mirage-git**
+  - Lower bound for `mirage-kv` is now `6.0.0` (#2256, @metanivek)
+
+- **irmin-pack**
+  - Removed `dict_auto_flush_threshold` and `suffix_auto_flush_threshold` configuration. (#2235, @art-w)
 
 ### Fixed
 
 - **irmin-cli**
   - Changed `--store irf` to `--store fs` to align the CLI with what is
-    published on the Irmin website (@wyn, #2243)
+    published on the Irmin website (#2243, @wyn)
+
+## 3.7.2 (2023-06-16)
 
-## 3.7.1 (2023-05-24)
+### Fixed
+
+- **irmin**
+  - Fix performance issue in `Tree.length` (#2258, @art-w)
 
+- **irmin-pack**
+  - Fix snapshot export when using lower layer (#2257, @metanivek)
+
+## 3.7.1 (2023-05-24)
 
 ### Fixed
 
 - **irmin-pack**
-  - Fix issue when migrating v2 stores to use lower layer (@metanivek, #2241)
+  - Fix issue when migrating v2 stores to use lower layer (#2241, @metanivek)
   - Fix issue when calling GC for a commit in the lower after migration
-    (@metanivek, #2242)
+    (#2242, @metanivek)
 
 ## 3.7.0 (2023-04-26)
 

CODE_OF_CONDUCT.md

@@ -0,0 +1,14 @@
+# Code of Conduct
+
+This project has adopted the [OCaml Code of Conduct](https://github.com/ocaml/code-of-conduct/blob/main/CODE_OF_CONDUCT.md).
+
+# Enforcement
+
+This project follows the OCaml Code of Conduct [enforcement policy](https://github.com/ocaml/code-of-conduct/blob/main/CODE_OF_CONDUCT.md#enforcement).
+
+To report any violations, please contact:
+Thomas Gazagnaire <thomas [at] tarides [dot] com>
+Arthur Wendling <arthur [at] tarides [dot] com>
+Kevin Smith <kevin [at] tarides [dot] com>
+Gwenaëlle Lecat <gwenaelle [at] tarides [dot] com>
+Vincent Balat <vincent.balat [at] tarides [dot] com>

README.md

@@ -36,7 +36,7 @@ rebase). It is highly customizable: users can define their types to
 store application-specific values and define custom storage layers (in
 memory, on disk, in a remote Redis database, in the browser,
 etc.). The Git workflow was initially designed for humans to manage
-changes within source code. Irmin scales this to hanlde automatic
+changes within source code. Irmin scales this to handle automatic
 programs performing a very high number of operations per second, with
 a fully automated handling of update conflicts. Finally, Irmin exposes
 an event-driven API to define programmable dynamic behaviours and to
@@ -143,7 +143,6 @@ The following packages have are available on `opam`:
 - `irmin-fs` - filesystem-based storage using `bin_prot`
 - `irmin-git` - Git compatible storage
 - `irmin-graphql` - GraphQL server
-- `irmin-http` - a simple REST interface
 - `irmin-mirage` - mirage compatibility
 - `irmin-mirage-git` - Git compatible storage for mirage
 - `irmin-mirage-graphql` - mirage compatible GraphQL server

doc/irmin-pack/version-history.md

@@ -0,0 +1,77 @@
+# Version History
+
+Summary of the data versions of `irmin-pack` and key changes for the entries in the pack file.
+
+## Data versions
+
+See `irmin-pack/version.ml`.
+
+| Version  | Description                      |
+| -------- | --------                         |
+| V1       | Original version of `irmin-pack` |
+| V2       | Introduced V2 version of objects/keys. [Irmin 3.0.0](https://github.com/mirage/irmin/releases/tag/3.0.0) via [PR#1655](https://github.com/mirage/irmin/pull/1655) |
+| V3       | Introduced control file. [Irmin 3.3.0](https://github.com/mirage/irmin/releases/tag/3.3.0) via [PR#1865](https://github.com/mirage/irmin/pull/1865) |
+| V4       | Introduced chunked suffix. [Irmin 3.5.0](https://github.com/mirage/irmin/releases/tag/3.5.0) via [PR#2110](https://github.com/mirage/irmin/pull/2110) |
+| V5       | Introduced lower layer + volumes. [Irmin 3.7.0](https://github.com/mirage/irmin/releases/tag/3.7.0) via [PR#2184](https://github.com/mirage/irmin/pull/2184) and [PR#2180](https://github.com/mirage/irmin/pull/2180) |
+
+Prior to `V3`, `irmin-pack` stored version information in headers of files but now stores it only in a header of the control file.
+
+## Pack store entries
+
+See `irmin-pack/pack_value.ml` and `irmin-pack/inode.ml` for how `irmin-pack` represents `irmin`'s core types of contents, nodes, and commits.
+
+### Contents
+
+| Version  | Description                                     |
+| -------- | --------                                        |
+| V1       | The first and only format for contents entries. |
+
+`Contents` objects only have one version, but a user-defined contents length header [configuration](https://github.com/mirage/irmin/blob/1f046ddaedf3532bb236f7d19510182c5948b9d6/src/irmin-pack/conf.mli#L34-L45) was added in [Irmin 3.0.0](https://github.com/mirage/irmin/releases/tag/3.0.0) via [PR#1644](https://github.com/mirage/irmin/pull/1644). 
+
+💡 Since Tezos' contents type is [bytes](https://github.com/mirage/irmin/blob/1f046ddaedf3532bb236f7d19510182c5948b9d6/src/irmin-tezos/schema.ml#L146) and `repr` stores [bytes](https://github.com/mirage/repr/blob/2b260367d8073a421e38be4e564d2c1931d15ee0/src/repr/type_intf.ml#L10-L11) with a [length prefix](https://github.com/mirage/repr/blob/2b260367d8073a421e38be4e564d2c1931d15ee0/src/repr/binary.ml#L307-L312) that is [LEB128-encoded](https://github.com/mirage/repr/blob/2b260367d8073a421e38be4e564d2c1931d15ee0/src/repr/binary.ml#L104-L112), its contents type automatically supports `Varint` for `contents_length_header`.
+
+### Inodes
+
+| Version  | Description                                         |
+| -------- | --------                                            |
+| V1       | Original data. Corresponds with V1 keys.            |
+| V2       | Introduces length prefix. Corresponds with V2 keys. |
+
+
+V1 added an [explicit length prefix](https://github.com/mirage/irmin/blob/1f046ddaedf3532bb236f7d19510182c5948b9d6/src/irmin-pack/inode.ml#L400) to support minimal indexing.
+
+### Commits
+
+| Version  | Description                                         |
+| -------- | --------                                            |
+| V1       | Original data. Corresponds with V1 keys.            |
+| V2       | Introduces length prefix. Corresponds with V2 keys. |
+
+V2 added an [explicit length prefix](https://github.com/mirage/irmin/blob/1f046ddaedf3532bb236f7d19510182c5948b9d6/src/irmin-pack/pack_value.ml#L193) to support minimal indexing.
+
+## Keys
+
+See `irmin-pack/unix/pack_key.ml`.
+
+| Version  | Name           | Description                                                           |
+| -------- | --------       | --------                                                              |
+| V1       | Hash           | Original "keys". Relied on index for all object offsets in pack file. |
+| V2       | Structured     | [Irmin 3.0.0](https://github.com/mirage/irmin/releases/tag/3.0.0) via [PR#1659](https://github.com/mirage/irmin/pull/1659). Supports existing `V0` through `Pack_key.Indexed`. All new keys are written as `Pack_key.Direct`. |
+
+Keys do not have versions in the code, but versions are indicated here to correspond to data version.
+
+## Indexing Strategies
+
+| Type        | Data Indexed                   | Description        |
+| --------    | --------                       | --------           |
+| Non-minimal | All: Contents, Inodes, Commits | Original strategy  |
+| Minimal     | Commits                        | [Irmin 3.0.0](https://github.com/mirage/irmin/releases/tag/3.0.0) via [PR#1664](https://github.com/mirage/irmin/pull/1664) |
+
+## Release Dates
+
+Dates of release are a lower bounds on when we can expect different versions of data in stores, but the actual date is later due to delays of when clients ship their updates.
+
+- [Irmin 3.0.0](https://github.com/mirage/irmin/releases/tag/3.0.0) on Feb 14, 2022.
+- [Irmin 3.3.0](https://github.com/mirage/irmin/releases/tag/3.3.0) on Jun 20, 2022.
+- [Irmin 3.5.0](https://github.com/mirage/irmin/releases/tag/3.5.0) on Dec 14, 2022.
+- [Irmin 3.7.0](https://github.com/mirage/irmin/releases/tag/3.5.0) on Apr 26, 2023.

examples/client_batch.ml

@@ -0,0 +1,47 @@
+(*
+ * Copyright (c) 2018-2022 Tarides <[email protected]>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *)
+
+open Lwt.Syntax
+open Lwt.Infix
+module Store = Irmin_mem.KV.Make (Irmin.Contents.String)
+module Client = Irmin_client_unix.Make (Store)
+module Error = Irmin_client.Error
+
+let main =
+  let info () = Client.Info.empty in
+  let uri = Uri.of_string Sys.argv.(1) in
+  let* client = Client.connect uri in
+
+  let* main = Client.main client in
+  let* () = Client.set_exn ~info main [ "testing" ] "testing" in
+  let* head = Client.Branch.get client Client.Branch.main in
+  let* tree =
+    Client.Batch.Tree.of_commit client (Client.Commit.hash head) >|= Option.get
+  in
+  let* tree = Client.Batch.Tree.add client tree [ "b"; "c" ] "123" in
+  let* tree = Client.Batch.Tree.add_tree client tree [ "a" ] tree in
+  let* tree = Client.Batch.Tree.remove client tree [ "testing" ] in
+  let* commit = Client.Batch.commit ~parents:[ head ] ~info client tree in
+  let* () = Client.Branch.set client Client.Branch.main commit in
+  let* foo = Client.get main [ "foo" ] in
+  let* abc = Client.get main [ "a"; "b"; "c" ] in
+  let* testing = Client.find main [ "testing" ] in
+  assert (foo = "bar");
+  assert (abc = "123");
+  assert (Option.is_none testing);
+  Lwt_io.printlf "foo => %s\na/b/c => %s" foo abc
+
+let () = Lwt_main.run main

examples/dune

@@ -10,7 +10,9 @@
   push
   custom_graphql
   custom_storage
-  fold)
+  fold
+  client_batch
+  server)
  (libraries
   astring
   cohttp
@@ -20,6 +22,10 @@
   irmin-graphql.unix
   irmin-pack.unix
   irmin-watcher
+  irmin-server.unix
+  irmin-client.unix
+  websocket-lwt-unix
+  conduit-lwt-unix
   lwt
   lwt.unix)
  (preprocess
@@ -39,7 +45,9 @@
   custom_graphql.exe
   fold.exe
   gc.exe
-  custom_storage.exe))
+  custom_storage.exe
+  client_batch.exe
+  server.exe))
 
 (alias
  (name runtest)

examples/irmin-pack/gc.ml

@@ -60,12 +60,6 @@ module Repo_config = struct
   (** Must use minimal indexing strategy to use GC *)
   let indexing_strategy = Irmin_pack.Indexing_strategy.minimal
 
-  (** Buffer size that triggers auto flushing to disk *)
-  let dict_auto_flush_threshold = 1_000_000
-
-  (** Buffer size that triggers auto flushing to disk *)
-  let suffix_auto_flush_threshold = 1_000_000
-
   (** Location on disk to save the repository
 
       Note: irmin-pack will not create the entire path, only the final directory *)
@@ -81,7 +75,7 @@ module Repo_config = struct
   (** Create config for our repository *)
   let config =
     Irmin_pack.config ~fresh ~index_log_size ~merge_throttle ~indexing_strategy
-      ~dict_auto_flush_threshold ~suffix_auto_flush_threshold root
+      root
 
   (** We can add an optional lower layer to our repository. Data discarded by
       the GC will be stored there and still be accessible instead of being
@@ -91,7 +85,7 @@ module Repo_config = struct
   (** Create a copy of the previous configuration, now with a lower layer *)
   let config_with_lower =
     Irmin_pack.config ~fresh ~index_log_size ~merge_throttle ~indexing_strategy
-      ~dict_auto_flush_threshold ~suffix_auto_flush_threshold ~lower_root root
+      ~lower_root root
 end
 
 (** Utility for creating commit info *)

examples/irmin-pack/kv.ml

@@ -50,12 +50,6 @@ module Repo_config = struct
   (** Must use minimal indexing strategy to use GC *)
   let indexing_strategy = Irmin_pack.Indexing_strategy.minimal
 
-  (** Buffer size that triggers auto flushing to disk *)
-  let dict_auto_flush_threshold = 1_000_000
-
-  (** Buffer size that triggers auto flushing to disk *)
-  let suffix_auto_flush_threshold = 1_000_000
-
   (** Location on disk to save the repository
 
       Note: irmin-pack will not create the entire path, only the final directory *)
@@ -71,7 +65,7 @@ module Repo_config = struct
   (** Create config for our repository *)
   let config =
     Irmin_pack.config ~fresh ~index_log_size ~merge_throttle ~indexing_strategy
-      ~dict_auto_flush_threshold ~suffix_auto_flush_threshold root
+      root
 end
 
 module StoreMaker = Irmin_pack_unix.KV (Conf)

examples/plugin/plugin.ml

@@ -1,3 +1,19 @@
+(*
+ * Copyright (c) 2018-2022 Tarides <[email protected]>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *)
+
 open Irmin_cli
 
 (* Adding a new content type *)

examples/server.ml

@@ -0,0 +1,40 @@
+(*
+ * Copyright (c) 2018-2022 Tarides <[email protected]>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *)
+
+open Lwt.Syntax
+module Store = Irmin_mem.KV.Make (Irmin.Contents.String)
+module Server = Irmin_server_unix.Make (Store)
+
+let info () = Irmin.Info.Default.empty
+
+let init () =
+  let* repo = Store.Repo.v (Irmin_mem.config ()) in
+  let* main = Store.main repo in
+  let+ () = Store.set_exn ~info main [ "foo" ] "bar" in
+  ()
+
+let main () =
+  let uri = Uri.of_string Sys.argv.(1) in
+  let config = Irmin_mem.config () in
+  let dashboard = `TCP (`Port 1234) in
+  let* server = Server.v ~uri ~dashboard config in
+  let () = Format.printf "Listening on %a@." Uri.pp uri in
+  Server.serve server
+
+let () =
+  Lwt_main.run
+  @@ let* () = init () in
+     main ()