feat: build ivf partition using disk based shuffler (#1312)

* Reduce the memory consumption to run IVF_PQ
* Building block for distributed IVF_PQ indexing
* Better multi-thread support during IVF assignment

benchmarks/wiki/index.py

@@ -13,12 +13,14 @@ def main():
         help="The metric type (L2, cosine, dot) to use for indexing",
     )
     parser.add_argument(
+        "-p",
         "--num-partitions",
         default=2048,
         type=int,
         help="The number of partitions to use for indexing",
     )
     parser.add_argument(
+        "-s",
         "--num-sub-vectors",
         default=96,
         type=int,

rust/lance/Cargo.toml

@@ -53,7 +53,6 @@ url = "2.3"
 rand = { version = "0.8.3", features = ["small_rng"] }
 futures = { workspace = true }
 uuid = { version = "1.2", features = ["v4"] }
-path-absolutize = "3.0.14"
 shellexpand = "3.0.0"
 arrow = { version = "43.0.0", features = ["prettyprint"] }
 num_cpus = "1.0"