Squash of 1222a14

A plain copy of memory kv store at commit c1f2ceb.

Renaming memory to zip
Manual modification: fix automatic renames and add to the build system
bzip2 was renamed into bz2lib In commit 07ec524.

Switch zarr unit tests to use zip kv store

Add implementation of reading and writing from file and memory

In-memory zip file creation inspired by code from:
https://github.com/zlib-ng/minizip-ng/blob/8bba5e3fdfc4f8a6c6033cf2b7656cfb2779bef1/test/test.c#L304-L411

Keep the zip open until a new filename is requested.

The key delete and update are not implemented.

Rename zip to zip_memory to avoid name conflict with the official one

tensorstore/driver/zarr/BUILD

@@ -130,6 +130,7 @@ tensorstore_cc_test(
         "//tensorstore/index_space:index_transform",
         "//tensorstore/internal/json_binding",
         "//tensorstore/kvstore/memory",
+        "//tensorstore/kvstore/zip_memory",
         "//tensorstore/util:span",
         "//tensorstore/util:status",
         "//tensorstore/util:status_testutil",

tensorstore/driver/zarr/driver_impl_test.cc

@@ -52,7 +52,7 @@ Result<tensorstore::IndexTransform<>> ResolveBoundsFromMetadata(
       auto store,
       tensorstore::Open({
                             {"driver", "zarr"},
-                            {"kvstore", {{"driver", "memory"}}},
+                            {"kvstore", {{"driver", "zip_memory"}}},
                             {"metadata", ::nlohmann::json(metadata)},
                             {"field", field},
                             {"create", true},
@@ -73,7 +73,7 @@ Result<ResizeParameters> GetResizeParameters(
       auto store,
       tensorstore::Open({
                             {"driver", "zarr"},
-                            {"kvstore", {{"driver", "memory"}}},
+                            {"kvstore", {{"driver", "zip_memory"}}},
                             {"metadata", ::nlohmann::json(metadata)},
                             {"field", field},
                             {"create", true},

tensorstore/kvstore/BUILD

@@ -12,6 +12,7 @@ DRIVER_DOCS = [
     "memory",
     "neuroglancer_uint64_sharded",
     "ocdbt",
+    "zip_memory",
     "zip",
 ]
 

tensorstore/kvstore/zip_memory/BUILD

@@ -0,0 +1,73 @@
+# Zip KeyValueStore driver
+
+load("//bazel:tensorstore.bzl", "tensorstore_cc_library", "tensorstore_cc_test")
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+filegroup(
+    name = "doc_sources",
+    srcs = glob([
+        "**/*.rst",
+        "**/*.yml",
+    ]),
+)
+
+tensorstore_cc_library(
+    name = "zip_memory",
+    srcs = ["zip_memory_key_value_store.cc"],
+    hdrs = ["zip_memory_key_value_store.h"],
+    deps = [
+        "//tensorstore:context",
+        "//tensorstore/internal:intrusive_ptr",
+        "//tensorstore/internal:uri_utils",
+        "//tensorstore/internal:path",
+        "//tensorstore/internal/json_binding",
+        "//tensorstore/internal/json_binding:bindable",
+        "//tensorstore/kvstore",
+        "//tensorstore/kvstore:byte_range",
+        "//tensorstore/kvstore:generation",
+        "//tensorstore/util:future",
+        "//tensorstore/util:result",
+        "//tensorstore/util:status",
+        "//tensorstore/util/execution",
+        "//tensorstore/util/execution:any_receiver",
+        "//tensorstore/util/execution:sender",
+        "@com_github_nlohmann_json//:nlohmann_json",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:btree",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/synchronization",
+        "@com_google_absl//absl/time",
+        "@minizip_ng//:minizip_ng",
+    ],
+    alwayslink = 1,
+)
+
+tensorstore_cc_test(
+    name = "zip_memory_key_value_store_test",
+    size = "small",
+    srcs = ["zip_memory_key_value_store_test.cc"],
+    deps = [
+        ":zip_memory",
+        "//tensorstore:context",
+        "//tensorstore/internal:json_gtest",
+        "//tensorstore/internal/cache_key",
+        "//tensorstore/kvstore",
+        "//tensorstore/kvstore:test_util",
+        "//tensorstore/serialization",
+        "//tensorstore/serialization:test_util",
+        "//tensorstore/util:future",
+        "//tensorstore/util:status",
+        "//tensorstore/util:status_testutil",
+        "//tensorstore/util/execution",
+        "//tensorstore/util/execution:sender",
+        "//tensorstore/util/execution:sender_testutil",
+        "@com_github_nlohmann_json//:nlohmann_json",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/strings:cord",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

tensorstore/kvstore/zip_memory/index.rst

@@ -0,0 +1,17 @@
+.. _zip_memory-kvstore-driver:
+
+``zip_memory`` Key-Value Store driver
+=================================
+
+The ``zip_memory`` driver stores key-value pairs in a zip file.
+The zip file can reside in memory or on the local file system.
+The main need for zip store is for use in JavaScript and WebAssembly,
+where a single file can be passed around as a blob,
+and a directory is very inconvenient (to say the least).
+
+.. json:schema:: kvstore/zip_memory
+
+.. json:schema:: Context.zip_memory_key_value_store
+
+.. json:schema:: KvStoreUrl/zip_memory
+

tensorstore/kvstore/zip_memory/schema.yml

@@ -0,0 +1,53 @@
+$schema: http://json-schema.org/draft-07/schema#
+$id: kvstore/zip_memory
+allOf:
+- $ref: KvStore
+- type: object
+  properties:
+    driver:
+      const: zip_memory
+    zip_memory_key_value_store:
+      $ref: ContextResource
+      description: |-
+        Specifies or references a previously defined
+        `Context.zip_memory_key_value_store`.
+    atomic:
+      type: boolean
+      default: true
+      description: |-
+        Support atomic multi-key transactions.  If set to ``false``, the
+        transaction behavior matches that of the `kvstore/file` and
+        `kvstore/gcs` drivers, which may be useful for testing purposes.
+definitions:
+  zip_memory_key_value_store:
+    $id: Context.zip_memory_key_value_store
+    description: |-
+      Represents the zip key-value store.  If multiple `kvstore/zip_memory`
+      specifications reference the same `Context.zip_memory_key_value_store`, they
+      all refer to the same zip file (set of key/value pairs).
+    type: object
+  url:
+    $id: KvStoreUrl/zip_memory
+    allOf:
+    - $ref: KvStoreUrl
+    - type: string
+    type: string
+    title: |
+      :literal:`zip_memory://` KvStore URL scheme
+    description: |
+      Zip key-value stores may be specified using the custom
+      :file:`zip_memory://{path}` URL syntax, and :file:`zip_memory:memory_address:length` syntax.
+
+      .. admonition:: Examples
+         :class: example
+
+         .. list-table::
+            :header-rows: 1
+            :widths: auto
+
+            * - URL representation
+              - JSON representation
+            * - ``"zip_memory://"``
+              - ``{"driver": "zip_memory"}``
+            * - ``"zip_memory://path/to/dataset"``
+              - ``{"driver": "zip_memory", "path": "path/to/dataset"}``