feat: Add support for multi-version metadata

As an example, for comparing against the most recent I've added the 5 most recent
vega · Oct 7, 2024 · 1792340 · 1792340
1 parent a618ffc
commit 1792340
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/tools/_vega_datasets_data/metadata_v2.5.4-v2.9.0.parquet b/tools/_vega_datasets_data/metadata_v2.5.4-v2.9.0.parquet
diff --git a/tools/vendor_datasets.py b/tools/vendor_datasets.py
@@ -135,6 +135,17 @@ def request_trees_to_df(tag: str, /) -> pl.DataFrame:
     return df.select(*sorted(df.columns))
 
 
+def request_trees_to_df_batched(*tags: str, delay: int = 5) -> pl.DataFrame:
+    import random
+    import time
+
+    dfs: list[pl.DataFrame] = []
+    for tag in tags:
+        time.sleep(delay + random.triangular())
+        dfs.append(request_trees_to_df(tag))
+    return pl.concat(dfs)
+
+
 def collect_metadata(tag: str, /, fp: Path, *, write_schema: bool = True) -> None:
     metadata = request_trees_to_df(tag)
     if not fp.exists():