diff --git a/package-lock.json b/package-lock.json index 01f7907..d8f9881 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3701,10 +3701,11 @@ } }, "node_modules/@ipld/dag-json": { - "version": "10.1.3", - "license": "Apache-2.0 OR MIT", + "version": "10.1.5", + "resolved": "https://registry.npmjs.org/@ipld/dag-json/-/dag-json-10.1.5.tgz", + "integrity": "sha512-AIIDRGPgIqVG2K1O42dPDzNOfP0YWV/suGApzpF+YWZLwkwdGVsxjmXcJ/+rwOhRGdjpuq/xQBKPCu1Ao6rdOQ==", "dependencies": { - "cborg": "^2.0.1", + "cborg": "^4.0.0", "multiformats": "^12.0.1" }, "engines": { @@ -3713,10 +3714,11 @@ } }, "node_modules/@ipld/dag-json/node_modules/cborg": { - "version": "2.0.3", - "license": "Apache-2.0", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-4.0.4.tgz", + "integrity": "sha512-nu+JXYskYqWN/tFWQVjL2ZYlUwK+dapqkTpruAtJkwmDv7XaTgg8PStUbO+sXfhqSWaeQ9LPSPCTrO2WZ2Bxfg==", "bin": { - "cborg": "cli.js" + "cborg": "lib/bin.js" } }, "node_modules/@ipld/dag-json/node_modules/multiformats": { @@ -7159,7 +7161,6 @@ }, "node_modules/buffer-from": { "version": "1.1.2", - "devOptional": true, "license": "MIT" }, "node_modules/buffer/node_modules/ieee754": { @@ -8871,7 +8872,6 @@ }, "node_modules/core-util-is": { "version": "1.0.3", - "devOptional": true, "license": "MIT" }, "node_modules/cpu-features": { @@ -9209,9 +9209,13 @@ "node": ">=4" } }, + "node_modules/duplex-maker": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/duplex-maker/-/duplex-maker-1.0.0.tgz", + "integrity": "sha512-KoHuzggxg7f+vvjqOHfXxaQYI1POzBm+ah0eec7YDssZmbt6QFBI8d1nl5GQwAgR2f+VQCPvyvZtmWWqWuFtlA==" + }, "node_modules/duplexify": { "version": "3.7.1", - "devOptional": true, "license": "MIT", "dependencies": { "end-of-stream": "^1.0.0", @@ -9268,7 +9272,6 @@ }, "node_modules/end-of-stream": { "version": "1.4.4", - "devOptional": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -11616,6 +11619,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-zst": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-zst/-/is-zst-1.0.0.tgz", + "integrity": "sha512-ZA5lvshKAl8z30dX7saXLpVhpsq3d2EHK9uf7qtUjnOtdw4XBpAoWb2RvZ5kyoaebdoidnGI0g2hn9Z7ObPbww==" + }, "node_modules/isarray": { "version": "2.0.5", "dev": true, @@ -11700,6 +11708,12 @@ "node": ">=4" } }, + "node_modules/json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "dev": true, @@ -12363,6 +12377,15 @@ "url": "https://github.com/sindresorhus/mem?sponsor=1" } }, + "node_modules/memorystream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", + "integrity": "sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==", + "dev": true, + "engines": { + "node": ">= 0.10.0" + } + }, "node_modules/merge-descriptors": { "version": "1.0.1", "dev": true, @@ -12638,6 +12661,12 @@ "node": ">= 0.6" } }, + "node_modules/nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, "node_modules/node-fetch": { "version": "2.6.12", "dev": true, @@ -12697,6 +12726,235 @@ "node": ">=0.10.0" } }, + "node_modules/npm-run-all": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/npm-run-all/-/npm-run-all-4.1.5.tgz", + "integrity": "sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "chalk": "^2.4.1", + "cross-spawn": "^6.0.5", + "memorystream": "^0.3.1", + "minimatch": "^3.0.4", + "pidtree": "^0.3.0", + "read-pkg": "^3.0.0", + "shell-quote": "^1.6.1", + "string.prototype.padend": "^3.0.0" + }, + "bin": { + "npm-run-all": "bin/npm-run-all/index.js", + "run-p": "bin/run-p/index.js", + "run-s": "bin/run-s/index.js" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/npm-run-all/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/npm-run-all/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, + "node_modules/npm-run-all/node_modules/cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "dependencies": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + }, + "engines": { + "node": ">=4.8" + } + }, + "node_modules/npm-run-all/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/npm-run-all/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha512-Kx8hMakjX03tiGTLAIdJ+lL0htKnXjEZN6hk/tozf/WOuYGdZBJrZ+rCJRbVCugsjB3jMLn9746NsQIf5VjBMw==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-aOIos8bujGN93/8Ox/jPLh7RwVnPEysynVFE+fQZyg6jKELEHwzgKdLRFHUgXJL6kylijVSBC4BvN9OmsB48Rw==", + "dev": true, + "dependencies": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/pidtree": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/pidtree/-/pidtree-0.3.1.tgz", + "integrity": "sha512-qQbW94hLHEqCg7nhby4yRC7G2+jYHY4Rguc2bjw7Uug4GIJuu1tvf2uHaZv5Q8zdt+WKJ6qK1FOI6amaWUo5FA==", + "dev": true, + "bin": { + "pidtree": "bin/pidtree.js" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/npm-run-all/node_modules/read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha512-BLq/cCO9two+lBgiTYNqD6GdtK8s4NpaWrl6/rCO9w0TUS8oJl7cmToOZfRYllKTISY6nt1U7jQ53brmKqY6BA==", + "dev": true, + "dependencies": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/npm-run-all/node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "dev": true, + "dependencies": { + "shebang-regex": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-all/node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-all/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, "node_modules/npm-run-path": { "version": "5.1.0", "dev": true, @@ -12871,7 +13129,6 @@ }, "node_modules/once": { "version": "1.4.0", - "devOptional": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -12997,6 +13254,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/p-defer": { "version": "1.0.0", "dev": true, @@ -13224,6 +13489,25 @@ "node": ">=8" } }, + "node_modules/peek-stream": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz", + "integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==", + "dependencies": { + "buffer-from": "^1.0.0", + "duplexify": "^3.5.0", + "through2": "^2.0.3" + } + }, + "node_modules/peek-stream/node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, "node_modules/picocolors": { "version": "1.0.0", "dev": true, @@ -13251,6 +13535,15 @@ "node": ">=0.10" } }, + "node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/pkg-conf": { "version": "4.0.0", "dev": true, @@ -13477,9 +13770,18 @@ }, "node_modules/process-nextick-args": { "version": "2.0.1", - "devOptional": true, "license": "MIT" }, + "node_modules/process-streams": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/process-streams/-/process-streams-1.0.1.tgz", + "integrity": "sha512-Z+FHhxiBhiQ4t/xTY3Bo2SxZG/CehflyckFsQirAXFRf/BfVnDePzpo58eq9JI4XfFu1RnX5C5EAE6V4sce1+g==", + "dependencies": { + "duplex-maker": "^1.0.0", + "quotemeta": "0.0.0", + "tempfile": "^1.1.0" + } + }, "node_modules/promptly": { "version": "3.2.0", "dev": true, @@ -13613,6 +13915,11 @@ ], "license": "MIT" }, + "node_modules/quotemeta": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/quotemeta/-/quotemeta-0.0.0.tgz", + "integrity": "sha512-1XGObUh7RN5b58vKuAsrlfqT+Rc4vmw8N4pP9gFCq1GFlTdV0Ex/D2Ro1Drvrqj++HPi3ig0Np17XPslELeMRA==" + }, "node_modules/range-parser": { "version": "1.2.1", "dev": true, @@ -13773,7 +14080,6 @@ }, "node_modules/readable-stream": { "version": "2.3.8", - "devOptional": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -13787,12 +14093,10 @@ }, "node_modules/readable-stream/node_modules/isarray": { "version": "1.0.0", - "devOptional": true, "license": "MIT" }, "node_modules/readable-stream/node_modules/safe-buffer": { "version": "5.1.2", - "devOptional": true, "license": "MIT" }, "node_modules/readdir-glob": { @@ -14239,6 +14543,15 @@ "node": ">=8" } }, + "node_modules/shell-quote": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.1.tgz", + "integrity": "sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/shelljs": { "version": "0.8.5", "dev": true, @@ -14282,6 +14595,17 @@ "simple-git-hooks": "cli.js" } }, + "node_modules/simple-zstd": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/simple-zstd/-/simple-zstd-1.4.2.tgz", + "integrity": "sha512-kGYEvT33M5XfyQvvW4wxl3eKcWbdbCc1V7OZzuElnaXft0qbVzoIIXHXiCm3JCUki+MZKKmvjl8p2VGLJc5Y/A==", + "dependencies": { + "is-zst": "^1.0.0", + "peek-stream": "^1.1.3", + "process-streams": "^1.0.1", + "through2": "^4.0.2" + } + }, "node_modules/slash": { "version": "4.0.0", "dev": true, @@ -14686,9 +15010,16 @@ "node": ">= 6" } }, + "node_modules/stream-read-all": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-4.0.0.tgz", + "integrity": "sha512-4MdJwfor9RkFCH1GCDCrEsLVqei+FrtogHtgyf2OdTlOq/+6+pW6FG1xzkdeK8/fd8/rGA7l3oJ1WolxNLX85w==", + "engines": { + "node": ">=12.20" + } + }, "node_modules/stream-shift": { "version": "1.0.1", - "devOptional": true, "license": "MIT" }, "node_modules/streamsearch": { @@ -14700,7 +15031,6 @@ }, "node_modules/string_decoder": { "version": "1.1.1", - "devOptional": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.1.0" @@ -14708,7 +15038,6 @@ }, "node_modules/string_decoder/node_modules/safe-buffer": { "version": "5.1.2", - "devOptional": true, "license": "MIT" }, "node_modules/string-argv": { @@ -14753,6 +15082,23 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/string.prototype.padend": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/string.prototype.padend/-/string.prototype.padend-3.1.5.tgz", + "integrity": "sha512-DOB27b/2UTTD+4myKUFh+/fXWcu/UDyASIXfg+7VzoCNNGOfWvoyU/x5pvVHr++ztyt/oSYI1BcWBBG/hmlNjA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.7", "dev": true, @@ -15074,6 +15420,24 @@ "node": ">=14.16" } }, + "node_modules/tempfile": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tempfile/-/tempfile-1.1.1.tgz", + "integrity": "sha512-NjT12fW6pSEKz1eVcADgaKfeM+XZ4+zSaqVz46XH7+CiEwcelnwtGWRRjF1p+xyW2PVgKKKS2UUw1LzRelntxg==", + "dependencies": { + "os-tmpdir": "^1.0.0", + "uuid": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tempfile/node_modules/uuid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "integrity": "sha512-FULf7fayPdpASncVy4DLh3xydlXEJJpvIELjYjNeQWYUZ9pclcpvCZSr2gkmN2FrrGcI7G/cJsIEwk5/8vfXpg==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details." + }, "node_modules/testcontainers": { "version": "9.12.0", "dev": true, @@ -15121,6 +15485,27 @@ "dev": true, "license": "MIT" }, + "node_modules/through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dependencies": { + "readable-stream": "3" + } + }, + "node_modules/through2/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/time-zone": { "version": "1.0.0", "dev": true, @@ -15495,7 +15880,6 @@ }, "node_modules/util-deprecate": { "version": "1.0.2", - "devOptional": true, "license": "MIT" }, "node_modules/utils-merge": { @@ -15769,7 +16153,6 @@ }, "node_modules/wrappy": { "version": "1.0.2", - "devOptional": true, "license": "ISC" }, "node_modules/write-file-atomic": { @@ -15835,7 +16218,6 @@ }, "node_modules/xtend": { "version": "4.0.2", - "devOptional": true, "license": "MIT", "engines": { "node": ">=0.4" @@ -15989,6 +16371,7 @@ "@aws-sdk/client-s3": "^3.363.0", "@aws-sdk/client-sqs": "^3.363.0", "@aws-sdk/util-dynamodb": "3.363.0", + "@ipld/dag-json": "10.1.5", "@ipld/dag-ucan": "^3.3.2", "@serverless-stack/node": "^1.18.4", "@ucanto/client": "^8.0.0", @@ -16005,6 +16388,8 @@ "@web3-storage/filecoin-client-legacy": "npm:@web3-storage/filecoin-client@^1.3.0", "multiformats": "12.0.1", "p-retry": "^5.1.2", + "simple-zstd": "^1.4.2", + "stream-read-all": "^4.0.0", "uint8arrays": "^4.0.4" }, "devDependencies": { @@ -16013,6 +16398,7 @@ "ava": "^5.3.0", "delay": "^6.0.0", "nanoid": "^4.0.0", + "npm-run-all": "^4.1.5", "p-defer": "^4.0.0", "p-wait-for": "^5.0.2", "sqs-consumer": "^7.2.2", diff --git a/package.json b/package.json index cbc05eb..ec4d736 100644 --- a/package.json +++ b/package.json @@ -57,10 +57,13 @@ "unicorn/explicit-length-check": "off", "unicorn/filename-case": "off", "unicorn/prefer-set-has": "off", + "unicorn/prefer-spread": "off", + "unicorn/prefer-array-some": "off", "unicorn/no-array-callback-reference": "off", "unicorn/no-array-reduce": "off", "unicorn/no-await-expression-member": "off", "unicorn/no-zero-fractions": "off", + "unicorn/numeric-separators-style": "off", "no-console": "off", "no-new": "off", "no-warning-comments": "off" diff --git a/packages/core/package.json b/packages/core/package.json index 8e67020..1b6a883 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -3,7 +3,10 @@ "version": "0.0.0", "type": "module", "scripts": { - "test": "ava --serial --no-worker-threads --verbose --timeout=60s test/{*.test.js,**/*.test.js}" + "mock:spade-oracle-server": "node test/helpers/spade-oracle-server.js", + "mock": "run-p mock:*", + "test": "PORT=9200 npm-run-all -p -r mock test:all", + "test:all": "ava --serial --no-worker-threads --verbose --timeout=60s test/{*.test.js,**/*.test.js}" }, "dependencies": { "@serverless-stack/node": "^1.18.4", @@ -11,6 +14,7 @@ "@aws-sdk/client-sqs": "^3.363.0", "@aws-sdk/client-s3": "^3.363.0", "@aws-sdk/util-dynamodb": "3.363.0", + "@ipld/dag-json": "10.1.5", "@ipld/dag-ucan": "^3.3.2", "@ucanto/client": "^8.0.0", "@ucanto/interface": "^8.0.0", @@ -26,13 +30,16 @@ "@web3-storage/filecoin-client": "^2.0.0", "multiformats": "12.0.1", "uint8arrays": "^4.0.4", - "p-retry": "^5.1.2" + "p-retry": "^5.1.2", + "simple-zstd": "^1.4.2", + "stream-read-all": "^4.0.0" }, "devDependencies": { "@ipld/car": "5.1.1", "@web-std/blob": "3.0.4", "ava": "^5.3.0", "nanoid": "^4.0.0", + "npm-run-all": "^4.1.5", "delay": "^6.0.0", "p-defer": "^4.0.0", "p-wait-for": "^5.0.2", diff --git a/packages/core/src/deal-tracker/spade-oracle-sync-tick.js b/packages/core/src/deal-tracker/spade-oracle-sync-tick.js new file mode 100644 index 0000000..608d826 --- /dev/null +++ b/packages/core/src/deal-tracker/spade-oracle-sync-tick.js @@ -0,0 +1,212 @@ +// @ts-expect-error no types available +import { ZSTDDecompress } from 'simple-zstd' +import { Readable } from 'stream' +// @ts-expect-error no types available +import streamReadAll from 'stream-read-all' +import { toString } from 'uint8arrays/to-string' +import { encode, decode } from '@ipld/dag-json' +import { RecordNotFoundErrorName } from '@web3-storage/filecoin-api/errors' +import { parse as parseLink } from 'multiformats/link' + +/** + * @typedef {import('@web3-storage/filecoin-api/deal-tracker/api').DealStore} DealStore + * @typedef {import('./types').OracleContracts} OracleContracts + * @typedef {import('./types').SpadeOracle} SpadeOracle + * @typedef {import('../store/types').SpadeOracleStore} SpadeOracleStore + */ + +/** + * On CRON tick, this function syncs deal store entries with the most up to date information stored + * in Spade's Oracle: + * - The previous oracle state known is fetched, as well as the latest state from Spade endpoint. + * - Once both states are in memory, they are compared and a diff is generated. + * - Diff is stored in deal store + * - Handled new state of oracle is stored for comparison in next tick. + * + * @param {object} context + * @param {DealStore} context.dealStore + * @param {SpadeOracleStore} context.spadeOracleStore + * @param {URL} context.spadeOracleUrl + */ +export async function spadeOracleSyncTick ({ + dealStore, + spadeOracleStore, + spadeOracleUrl +}) { + // Get previous recorded spade oracle contracts + const getPreviousSpadeOracle = await getSpadeOracleState({ + spadeOracleStore, + spadeOracleId: spadeOracleUrl.toString(), + }) + if (getPreviousSpadeOracle.error && getPreviousSpadeOracle.error.name !== RecordNotFoundErrorName) { + return getPreviousSpadeOracle + } + + // Get updated spade oracle contracts + const getUpdatedSpadeOracle = await getSpadeOracleCurrentState(spadeOracleUrl) + if (getUpdatedSpadeOracle.error) { + return getUpdatedSpadeOracle + } + + // Get diff of contracts + const diffOracleContracts = computeDiffOracleState({ + // fallsback to empty map if not found + previousOracleContracts: getPreviousSpadeOracle.ok || new Map(), + updatedOracleContracts: getUpdatedSpadeOracle.ok + }) + + // Store diff of contracts + const putDiff = await putDiffToDealStore({ + dealStore, + diffOracleContracts + }) + if (putDiff.error) { + return putDiff + } + + // Record spade oracle contracts handled + const putUpdatedSpadeOracle = await putUpdatedSpadeOracleState({ + spadeOracleStore, + spadeOracleId: spadeOracleUrl.toString(), + oracleContracts: getUpdatedSpadeOracle.ok + }) + if (putUpdatedSpadeOracle.error) { + return putUpdatedSpadeOracle + } + + return { + ok: {}, + error: undefined + } +} + +/** + * @param {object} context + * @param {DealStore} context.dealStore + * @param {OracleContracts} context.diffOracleContracts + * @returns {Promise>} + */ +export async function putDiffToDealStore ({ dealStore, diffOracleContracts }) { + const res = await Promise.all( + Array.from(diffOracleContracts, ([key, value]) => { + return Promise.all(value.map(contract => { + /** @type {import('@web3-storage/data-segment').LegacyPieceLink} */ + const legacyPieceCid = parseLink(key) + + return dealStore.put({ + ...contract, + // @ts-expect-error not PieceCIDv2 + piece: legacyPieceCid, + provider: `${contract.provider}`, + insertedAt: (new Date()).toISOString() + }) + })) + }) + ) + + const firsPutError = res.find(pieceContracts => pieceContracts.find(c => c.error))?.find(comb => comb.error) + if (firsPutError?.error) { + return { + error: firsPutError.error + } + } + return { + ok: {} + } +} + +/** + * @param {object} context + * @param {OracleContracts} context.previousOracleContracts + * @param {OracleContracts} context.updatedOracleContracts + */ +export function computeDiffOracleState ({ previousOracleContracts, updatedOracleContracts }) { + /** @type {OracleContracts} */ + const diff = new Map() + + for (const [pieceCid, contracts] of updatedOracleContracts.entries() ) { + const previousContracts = previousOracleContracts.get(pieceCid) || [] + // Find diff when different length + if (contracts.length !== previousContracts.length) { + const diffContracts = [] + // Get contracts for PieceCID still not recorded + for (const c of contracts) { + if (!previousContracts.find(pc => pc.dealId === c.dealId)) { + diffContracts.push(c) + } + } + diff.set(pieceCid, diffContracts) + } + } + + return diff +} + +/** + * @param {object} context + * @param {SpadeOracleStore} context.spadeOracleStore + * @param {string} context.spadeOracleId + * @returns {Promise>} + */ +export async function getSpadeOracleState ({ spadeOracleStore, spadeOracleId }) { + const getRes = await spadeOracleStore.get(spadeOracleId) + if (getRes.error) { + return getRes + } + + return { + ok: new Map(Object.entries(decode(getRes.ok.value))), + } +} + +/** + * @param {object} context + * @param {SpadeOracleStore} context.spadeOracleStore + * @param {string} context.spadeOracleId + * @param {OracleContracts} context.oracleContracts + */ +async function putUpdatedSpadeOracleState ({ spadeOracleStore, spadeOracleId, oracleContracts }) { + const putRes = await spadeOracleStore.put({ + key: spadeOracleId, + value: encode(Object.fromEntries(oracleContracts)) + }) + + return putRes +} + +/** + * @param {URL} spadeOracleUrl + * @returns {Promise>} + */ +async function getSpadeOracleCurrentState (spadeOracleUrl) { + /** @type {OracleContracts} */ + const dealMap = new Map() + const res = await fetch(spadeOracleUrl) + if (!res.ok) { + return { + // TODO: Error + error: new Error('could not read') + } + } + + const resDecompressed = await streamReadAll( + // @ts-expect-error aws types... + Readable.fromWeb(res.body) + .pipe(ZSTDDecompress()) + ) + /** @type {SpadeOracle} */ + const SpadeOracle = JSON.parse(toString(resDecompressed)) + for (const replica of SpadeOracle.active_replicas) { + // TODO: convert pieceCid to v2 (piece_cid) + (piece_log2_size) + dealMap.set(replica.piece_cid, replica.contracts.map(c => ({ + provider: c.provider_id, + dealId: c.legacy_market_id, + expirationEpoch: c.legacy_market_end_epoch, + source: spadeOracleUrl.toString() + }))) + } + + return { + ok: dealMap + } +} diff --git a/packages/core/src/deal-tracker/types.ts b/packages/core/src/deal-tracker/types.ts new file mode 100644 index 0000000..eedd0df --- /dev/null +++ b/packages/core/src/deal-tracker/types.ts @@ -0,0 +1,28 @@ + +export type OracleContracts = Map + +export interface Contract { + provider: number + dealId: number + expirationEpoch: number + source: string +} + +// Spade types + +export interface SpadeContract { + provider_id: number + legacy_market_id: number + legacy_market_end_epoch: number +} + +export interface SpadeReplica { + contracts: SpadeContract[] + piece_cid: string + piece_log2_size: number +} + +export interface SpadeOracle { + state_epoch: number + active_replicas: SpadeReplica[] +} diff --git a/packages/core/src/store/spade-oracle-store.js b/packages/core/src/store/spade-oracle-store.js new file mode 100644 index 0000000..957c23f --- /dev/null +++ b/packages/core/src/store/spade-oracle-store.js @@ -0,0 +1,98 @@ +import { PutObjectCommand, GetObjectCommand } from '@aws-sdk/client-s3' +import pRetry from 'p-retry' +import { StoreOperationFailed, RecordNotFound } from '@web3-storage/filecoin-api/errors' + +import { connectBucket } from './index.js' + + +/** + * @param {import('./types.js').BucketConnect | import('@aws-sdk/client-s3').S3Client} conf + * @param {object} context + * @param {string} context.name + * @returns {import('./types.js').SpadeOracleStore} + */ +export function createClient (conf, context) { + const bucketClient = connectBucket(conf) + + return { + put: async (record) => { + const putCmd = new PutObjectCommand({ + Bucket: context.name, + Key: encodeURIComponent(record.key), + Body: record.value + }) + + // retry to avoid throttling errors + try { + await pRetry(() => bucketClient.send(putCmd)) + } catch (/** @type {any} */ error) { + console.log('err', error) + return { + error: new StoreOperationFailed(error.message) + } + } + + return { + ok: {} + } + }, + get: async (key) => { + const putCmd = new GetObjectCommand({ + Bucket: context.name, + Key: encodeURIComponent(key) + }) + + let res + try { + res = await bucketClient.send(putCmd) + } catch (/** @type {any} */ error) { + if (error?.$metadata.httpStatusCode === 404) { + return { + error: new RecordNotFound('item not found in store') + } + } + return { + error: new StoreOperationFailed(error.message) + } + } + + if (!res || !res.Body) { + return { + error: new RecordNotFound('item not found in store') + } + } + + return { + ok: { + key, + value: await res.Body.transformToByteArray() + } + } + }, + has: async (key) => { + const putCmd = new GetObjectCommand({ + Bucket: context.name, + Key: encodeURIComponent(key) + }) + + let res + try { + res = await bucketClient.send(putCmd) + } catch (/** @type {any} */ error) { + return { + error: new StoreOperationFailed(error.message) + } + } + + if (!res || !res.Body) { + return { + error: new RecordNotFound('item not found in store') + } + } + + return { + ok: true + } + }, + } +} diff --git a/packages/core/src/store/types.ts b/packages/core/src/store/types.ts index 8be32f7..068cc3b 100644 --- a/packages/core/src/store/types.ts +++ b/packages/core/src/store/types.ts @@ -1,3 +1,7 @@ +import { Store } from '@web3-storage/filecoin-api/types' +import { ByteView } from '@ucanto/interface' +import { Contract } from '../deal-tracker/types' + // Connectors export interface BucketConnect { @@ -8,7 +12,17 @@ export interface TableConnect { region: string } +// Stores + // Store records +export interface SpadeOracleRecord { + key: string + value: ByteView<{ + [k: string]: Contract[]; + }> +} + +export type SpadeOracleStore = Store export interface DealStoreRecord { // PieceCid of an Aggregate `bagy...aggregate` diff --git a/packages/core/test/deal-tracker/spade-oracle-sync.test.js b/packages/core/test/deal-tracker/spade-oracle-sync.test.js new file mode 100644 index 0000000..9b57d62 --- /dev/null +++ b/packages/core/test/deal-tracker/spade-oracle-sync.test.js @@ -0,0 +1,264 @@ +import { testDealTracker as test } from '../helpers/context.js' +import { createS3, createBucket, createDynamodDb, createTable } from '../helpers/resources.js' + +import { parse as parseLink } from 'multiformats/link' +import { encode, decode } from '@ipld/dag-json' + +import { dealStoreTableProps } from '../../src/store/index.js' +import { createClient as createDealStoreClient } from '../../src/store/deal-store.js' +import { createClient as createSpadeOracleStoreClient } from '../../src/store/spade-oracle-store.js' +import * as spadeOracleSyncTick from '../../src/deal-tracker/spade-oracle-sync-tick.js' +import { RecordNotFoundErrorName } from '@web3-storage/filecoin-api/errors' + +test.beforeEach(async (t) => { + const dynamo = await createDynamodDb() + Object.assign(t.context, { + s3: (await createS3()).client, + dynamoClient: dynamo.client, + }) +}) + +test('downloads spade oracle replicas file from http server', async t => { + const { + dealStore, + spadeOracleStore, + spadeOracleUrl + } = await getContext(t.context) + + const spadeOracleSyncTickHandle = await spadeOracleSyncTick.spadeOracleSyncTick({ + dealStore, + spadeOracleStore, + spadeOracleUrl + }) + t.truthy(spadeOracleSyncTickHandle.ok) + t.falsy(spadeOracleSyncTickHandle.error) + + // Verify spade oracle was stored for future check + const getSpadeOracle = await spadeOracleStore.get(spadeOracleUrl.toString()) + if (getSpadeOracle.error) { + throw new Error('could get get spade oracle stored') + } + t.truthy(getSpadeOracle.ok) + + /** @type {import('../../src/deal-tracker/types.js').OracleContracts} */ + const oracleState = new Map(Object.entries(decode(getSpadeOracle.ok.value))) + t.is(oracleState.size, 2) + + // Verify deals were stored + for (const [pieceCid, contracts] of oracleState.entries()) { + t.is(contracts.length, 2) + for (const c of contracts) { + const piece = parseLink(pieceCid) + const getDealEntry = await dealStore.get({ + // @ts-expect-error old piece CID + piece, + dealId: c.dealId + }) + if (getDealEntry.error) { + throw new Error('could get get deal entry stored') + } + + t.truthy(getDealEntry.ok) + t.truthy(piece.equals(getDealEntry.ok.piece)) + t.is(c.source, getDealEntry.ok.source) + t.is(c.dealId, getDealEntry.ok.dealId) + t.is(String(c.provider), getDealEntry.ok.provider) + t.is(c.expirationEpoch, getDealEntry.ok.expirationEpoch) + t.truthy(getDealEntry.ok.insertedAt) + } + } +}) + +test('gets previous spade oracle state if available', async t => { + const { + spadeOracleStore, + spadeOracleUrl + } = await getContext(t.context) + const spadeOracleId = spadeOracleUrl.toString() + const source = encodeURIComponent(spadeOracleUrl.toString()) + const oracleContracts = getOracleContracts(source) + + // Try to get init state + const getSpadeOracleStateInit = await spadeOracleSyncTick.getSpadeOracleState({ + spadeOracleStore, + spadeOracleId, + }) + t.falsy(getSpadeOracleStateInit.ok) + t.truthy(getSpadeOracleStateInit.error) + t.is(getSpadeOracleStateInit.error?.name, RecordNotFoundErrorName) + + // Populate store with first diff + const putSpadeOracleState = await spadeOracleStore.put({ + key: spadeOracleId, + value: encode(Object.fromEntries(oracleContracts)) + }) + t.truthy(putSpadeOracleState.ok) + + // Get state and validate + const getSpadeOracleState = await spadeOracleSyncTick.getSpadeOracleState({ + spadeOracleStore, + spadeOracleId, + }) + if (getSpadeOracleState.error) { + throw new Error('could get get spade oracle stored') + } + t.truthy(getSpadeOracleState.ok) + t.is(getSpadeOracleState.ok.size, oracleContracts.size) + for (const [pieceCid, contracts] of getSpadeOracleState.ok.entries()) { + const insertedContracts = oracleContracts.get(pieceCid) + t.truthy(insertedContracts) + t.deepEqual(insertedContracts, contracts) + } +}) + +test('computes diff', async t => { + const { + dealStore, + spadeOracleUrl + } = await getContext(t.context) + + // Get previous oracle contracts + const source = encodeURIComponent(spadeOracleUrl.toString()) + const previousOracleContracts = getOracleContracts(source) + + // Store diff as previous oracle contracts + const putDiffInit = await spadeOracleSyncTick.putDiffToDealStore({ + dealStore, + diffOracleContracts: previousOracleContracts + }) + t.truthy(putDiffInit.ok) + + // Verify pieces before updates + const alreadyExistingPieceCid = 'baga6ea4seaqhmw7z7q3jypdr54xaluhzdn6syn7ovovvjpaqul2qqenhmg43wii' + const newPieceCid = 'baga6ea4seaqlskmw3rlwyebtplguyvr7rmuofydmnud2o6a5soyydgcede56kkq' + const queryInitAlreadyExisting = await dealStore.query({ + piece: parseLink(alreadyExistingPieceCid) + }) + t.truthy(queryInitAlreadyExisting.ok) + t.is(queryInitAlreadyExisting.ok?.length, 2) + + const queryInitNew = await dealStore.query({ + piece: parseLink(newPieceCid) + }) + t.truthy(queryInitNew.ok) + t.is(queryInitNew.ok?.length, 0) + + // Create updated oracle + const updatedOracleContracts = getOracleContracts(source) + // Add one more contract to one existing item + const itemToAddContract = updatedOracleContracts.get(alreadyExistingPieceCid) || [] + itemToAddContract?.push({ + provider: 2095199, + dealId: 40745773, + expirationEpoch: 4477915, + source + }) + updatedOracleContracts.set(alreadyExistingPieceCid, itemToAddContract) + + // Add new piece contracts + updatedOracleContracts.set( + newPieceCid, + [ + { + provider: 20378, + dealId: 41028500, + expirationEpoch: 4482396, + source + } + ] + ) + + const diffOracleContracts = spadeOracleSyncTick.computeDiffOracleState({ + previousOracleContracts, + updatedOracleContracts + }) + t.is(diffOracleContracts.size, 2) + // 1 new item in each to update + t.truthy(diffOracleContracts.get(alreadyExistingPieceCid)?.length === 1) + t.truthy(diffOracleContracts.get(newPieceCid)?.length === 1) + + // Store diff + const putDiffResult = await spadeOracleSyncTick.putDiffToDealStore({ + dealStore, + diffOracleContracts + }) + t.truthy(putDiffResult.ok) + + const queryAfterDiffAlreadyExisting = await dealStore.query({ + piece: parseLink(alreadyExistingPieceCid) + }) + t.truthy(queryAfterDiffAlreadyExisting.ok) + t.is(queryAfterDiffAlreadyExisting.ok?.length, 3) + + const queryAfterDiffNew = await dealStore.query({ + piece: parseLink(newPieceCid) + }) + t.truthy(queryAfterDiffNew.ok) + t.is(queryAfterDiffNew.ok?.length, 1) +}) + +/** + * @param {string} source + */ +function getOracleContracts (source) { + /** @type {import('../../src/deal-tracker/types.js').OracleContracts} */ + const oracleContracts = new Map() + oracleContracts.set( + 'baga6ea4seaqhmw7z7q3jypdr54xaluhzdn6syn7ovovvjpaqul2qqenhmg43wii', + [ + { + provider: 2095132, + dealId: 40745772, + expirationEpoch: 4477915, + source + }, + { + provider: 20378, + dealId: 41028577, + expirationEpoch: 4482396, + source + } + ] + ) + oracleContracts.set( + 'baga6ea4seaqgmg7ogugsyopv4gkbrgugiaq5mn6kofngpbot4gulgllv5q3kmei', + [ + { + provider: 2095132, + dealId: 38117821, + expirationEpoch: 4429240, + source + }, + { + provider: 1784458, + dealId: 46363131, + expirationEpoch: 4580799, + source + } + ] + ) + + return oracleContracts +} + +/** + * @param {import('../helpers/context.js').BucketContext & import('../helpers/context.js').DbContext} context + */ +async function getContext (context) { + const { s3, dynamoClient } = context + const bucketName = await createBucket(s3) + const tableName = await createTable(dynamoClient, dealStoreTableProps) + + const dealStore = createDealStoreClient(dynamoClient, { + tableName + }) + const spadeOracleStore = createSpadeOracleStoreClient(s3, { + name: bucketName + }) + + return { + dealStore, + spadeOracleStore, + spadeOracleUrl: new URL(`http://127.0.0.1:${process.env.PORT || 9000}`) + } +} diff --git a/packages/core/test/fixtures/active_replicas.json.zst b/packages/core/test/fixtures/active_replicas.json.zst new file mode 100644 index 0000000..04a9e21 Binary files /dev/null and b/packages/core/test/fixtures/active_replicas.json.zst differ diff --git a/packages/core/test/helpers/context.js b/packages/core/test/helpers/context.js index f587fd2..2afd792 100644 --- a/packages/core/test/helpers/context.js +++ b/packages/core/test/helpers/context.js @@ -23,6 +23,7 @@ import anyTest from 'ava' * @typedef {import('ava').TestFn} TestStore * @typedef {import('ava').TestFn} TestQueue * @typedef {import('ava').TestFn} TestWorkflow + * @typedef {import('ava').TestFn} TestDealTracker * @typedef {import('ava').TestFn} TestWorkflowWithMultipleQueues */ @@ -41,5 +42,8 @@ export const testService = /** @type {TestService} */ (anyTest) // eslint-disable-next-line unicorn/prefer-export-from export const tesWorkflow = /** @type {TestWorkflow} */ (anyTest) +// eslint-disable-next-line unicorn/prefer-export-from +export const testDealTracker = /** @type {TestDealTracker} */ (anyTest) + // eslint-disable-next-line unicorn/prefer-export-from export const tesWorkflowWithMultipleQueues = /** @type {TestWorkflowWithMultipleQueues} */ (anyTest) diff --git a/packages/core/test/helpers/spade-oracle-server.js b/packages/core/test/helpers/spade-oracle-server.js new file mode 100644 index 0000000..3ea7ba1 --- /dev/null +++ b/packages/core/test/helpers/spade-oracle-server.js @@ -0,0 +1,21 @@ +import { createServer } from 'http' +import fs from 'fs' +import path from 'path' +import { fileURLToPath } from 'url' + +const port = process.env.PORT ?? 9000 +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const fixtureName = process.env.FIXTURE_NAME || 'active_replicas.json.zst' + +const server = createServer((req, res) => { + fs.readFile(path.resolve(`${__dirname}`, '..', 'fixtures', fixtureName), (error, content) => { + if (error) { + res.writeHead(500) + res.end() + } + res.writeHead(200, { 'Content-disposition': 'attachment; filename=' + fixtureName }) + res.end(content) + }) +}) + +server.listen(port, () => console.log(`Listening on :${port}`)) diff --git a/packages/functions/src/deal-tracker/spade-oracle-sync-tick.js b/packages/functions/src/deal-tracker/spade-oracle-sync-tick.js new file mode 100644 index 0000000..902c50c --- /dev/null +++ b/packages/functions/src/deal-tracker/spade-oracle-sync-tick.js @@ -0,0 +1,67 @@ +import * as Sentry from '@sentry/serverless' +import { Table } from 'sst/node/table' + +import { createClient as createSpadeOracleStoreClient } from '@w3filecoin/core/src/store/spade-oracle-store.js' +import { createClient as createDealStoreClient } from '@w3filecoin/core/src/store/deal-store.js' +import { spadeOracleSyncTick } from '@w3filecoin/core/src/deal-tracker/spade-oracle-sync-tick.js' + +import { mustGetEnv } from '../utils.js' + +Sentry.AWSLambda.init({ + environment: process.env.SST_STAGE, + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 1.0, +}) + +export async function main() { + // Construct context + const { + spadeOracleStoreBucketName, + spadeOracleStoreBucketRegion, + dealStoreTableName, + dealStoreTableRegion, + spadeOracleUrl + } = getLambdaEnv() + + const spadeOracleStore = createSpadeOracleStoreClient({ + region: spadeOracleStoreBucketRegion + }, { + name: spadeOracleStoreBucketName + }) + const dealStore = createDealStoreClient({ + region: dealStoreTableRegion + }, { + tableName: dealStoreTableName.tableName + }) + + const { ok, error } = await spadeOracleSyncTick({ + dealStore, + spadeOracleStore, + spadeOracleUrl: new URL(spadeOracleUrl) + }) + + if (error) { + return { + statusCode: 500, + body: error.message + } + } + + return { + statusCode: 200, + body: ok + } +} + +/** + * Get Env validating it is set. + */ +function getLambdaEnv () { + return { + spadeOracleStoreBucketName: mustGetEnv('SPADE_ORACLE_STORE_BUCKET_NAME'), + spadeOracleStoreBucketRegion: mustGetEnv('SPADE_ORACLE_STORE_REGION'), + dealStoreTableName: Table['deal-store'], + dealStoreTableRegion: mustGetEnv('AWS_REGION'), + spadeOracleUrl: mustGetEnv('SPADE_ORACLE_URL'), + } +} diff --git a/sst.config.js b/sst.config.js index 10c2fd5..6ad982b 100644 --- a/sst.config.js +++ b/sst.config.js @@ -3,6 +3,7 @@ import { Tags } from 'aws-cdk-lib'; import { ApiStack } from './stacks/api-stack.js' import { DataStack } from './stacks/data-stack.js' import { ProcessorStack } from './stacks/processor-stack.js' +import { DealTrackerStack } from './stacks/deal-tracker-stack.js' export default { config() { @@ -30,6 +31,7 @@ export default { app .stack(DataStack) .stack(ProcessorStack) + .stack(DealTrackerStack) .stack(ApiStack) // tags let us discover all the aws resource costs incurred by this app diff --git a/stacks/data-stack.js b/stacks/data-stack.js index 27e113a..6f388da 100644 --- a/stacks/data-stack.js +++ b/stacks/data-stack.js @@ -75,7 +75,7 @@ export function DataStack({ stack, app }) { const spaceOracleStoreBucket = new Bucket(stack, spaceOracleBucket.bucketName, { cors: true, cdk: { - bucket + bucket: spaceOracleBucket } }) diff --git a/stacks/deal-tracker-stack.js b/stacks/deal-tracker-stack.js new file mode 100644 index 0000000..68feb2d --- /dev/null +++ b/stacks/deal-tracker-stack.js @@ -0,0 +1,55 @@ +import { Cron, use } from 'sst/constructs' + +import { DataStack } from './data-stack.js' +import { + setupSentry, + getDealTrackerEnv, + getResourceName +} from './config.js' + +/** + * @param {import('sst/constructs').StackContext} properties + */ +export function DealTrackerStack({ stack, app }) { + const { + SPADE_ORACLE_URL + } = getDealTrackerEnv() + + // Setup app monitoring with Sentry + setupSentry(app, stack) + + const { + dealStoreTable, + spaceOracleStoreBucket + } = use(DataStack) + + /** + * CRON to track deals resolution from Spade Oracle + */ + const spadeOracleCronName = getResourceName('spade-oracle-sync-cron', stack.stage) + const spadeOracleCron = new Cron(stack, spadeOracleCronName, { + // Spade updates each hour + schedule: 'rate(1 hour)', + job: { + function: { + handler: 'packages/functions/src/deal-tracker/spade-oracle-sync-tick.main', + memorySize: '1 GB', + environment: { + SPADE_ORACLE_URL, + SPADE_ORACLE_STORE_BUCKET_NAME: spaceOracleStoreBucket.bucketName, + SPADE_ORACLE_STORE_REGION: stack.region, + }, + bind: [ + dealStoreTable + ], + permissions: [ + spaceOracleStoreBucket + ] + } + } + }) + + return { + spadeOracleCron + } +}