From 425c53f425d6615c2a8e107867c7fb9ccf25a150 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 7 Jun 2024 14:32:13 +0100 Subject: [PATCH 01/33] Handle 404 for /zarr.json AND /.zattrs --- src/App.svelte | 6 +++--- src/utils.js | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/App.svelte b/src/App.svelte index a62e5dd..6d2131c 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -6,7 +6,7 @@ import Modal from "svelte-simple-modal"; import SplashScreen from "./SplashScreen.svelte"; - import { getJson } from "./utils"; + import { getZarrJson } from "./utils"; import CheckMark from "./CheckMark.svelte"; const searchParams = new URLSearchParams(window.location.search); @@ -21,8 +21,8 @@ if (source) { // load JSON to be validated... - console.log("Loading JSON... " + source + "/.zattrs"); - promise = getJson(source + "/.zattrs"); + console.log("Loading JSON... " + source); + promise = getZarrJson(source); } diff --git a/src/utils.js b/src/utils.js index 1008304..b45827d 100644 --- a/src/utils.js +++ b/src/utils.js @@ -3,6 +3,7 @@ import Ajv from "ajv"; export const CURRENT_VERSION = "0.4"; +export const FILE_NOT_FOUND = "File not found"; const ajv = new Ajv({ strict: false }); // options can be passed, e.g. {allErrors: true} @@ -18,7 +19,14 @@ async function fetchHandleError(url) { rsp = await fetch(url).then(function (response) { if (!response.ok) { // make the promise be rejected if we didn't get a 2xx response - msg += ` ${response.statusText}`; + console.log("response.statusText", response.statusText, 'statusCode', response.status) + // NB. statusText could be "Not Found" or "File not found" depending on server + // Standardise based on response.status + if (response.status == 404) { + msg += ` ${FILE_NOT_FOUND}`; + } else { + msg += ` ${response.statusText}`; + } } else { return response; } @@ -44,6 +52,36 @@ async function fetchHandleError(url) { throw Error(msg); } + +export async function getZarrJson(zarr_dir) { + let zarrJson; + let msg; + // try to load v2 /.zattrs or v3 /zarr.json + try { + zarrJson = await getJson(zarr_dir + "/zarr.json"); + } catch (error) { + console.log("getZarrJson", error) + if (!error.message.includes(FILE_NOT_FOUND)) { + throw error; + } + // IF we got a 404 then try other URL + try { + zarrJson = await getJson(zarr_dir + "/.zattrs"); + } catch (err2) { + if (err2.message.includes(FILE_NOT_FOUND)) { + throw Error(`No .zattrs or zarr.json at ${zarr_dir}: ${FILE_NOT_FOUND}`); + } else { + // First error was 404 but this isn't... + throw err2; + } + } + + } + if (zarrJson) { + return zarrJson; + } +} + export async function getJson(url) { return fetchHandleError(url).then((rsp) => rsp.json()); } From d7093528b73dd58dd4374ea7a0f81e4e9e0548f0 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 7 Jun 2024 16:49:29 +0100 Subject: [PATCH 02/33] Load v0.5 schemas from github PR branch --- src/JsonValidator/index.svelte | 2 +- src/utils.js | 65 ++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/src/JsonValidator/index.svelte b/src/JsonValidator/index.svelte index de70a83..7fdbd1e 100644 --- a/src/JsonValidator/index.svelte +++ b/src/JsonValidator/index.svelte @@ -42,7 +42,7 @@ Using schema{schemaUrls.length > 1 ? "s" : ""}: {#each schemaUrls as url, i} {i > 0 ? " and " : ""} - {url.split("main")[1]} + {url} {/each} {#await promise} diff --git a/src/utils.js b/src/utils.js index b45827d..17ae143 100644 --- a/src/utils.js +++ b/src/utils.js @@ -2,15 +2,30 @@ import Ajv from "ajv"; -export const CURRENT_VERSION = "0.4"; +export const CURRENT_VERSION = "0.5"; +// versions to check for e.g. attributes['https://ngff.openmicroscopy.org/0.5'] +// which start at version 0.5 +export const NAMESPACED_VERSIONS = ["0.5"] export const FILE_NOT_FOUND = "File not found"; const ajv = new Ajv({ strict: false }); // options can be passed, e.g. {allErrors: true} export function getSchemaUrl(schemaName, version) { + if (version == "0.5") { + // TEMP: use open PR branch + return `https://raw.githubusercontent.com/normanrz/ngff/spec-rfc2/latest/schemas/${schemaName}.schema`; + } return `https://raw.githubusercontent.com/ome/ngff/main/${version}/schemas/${schemaName}.schema`; } +function getNamespacedKey(version) { + if (!version) { + version = CURRENT_VERSION; + } + return `https://ngff.openmicroscopy.org/${version}`; +} + + // fetch() doesn't error for 404 etc. async function fetchHandleError(url) { let msg = `Error Loading ${url}:`; @@ -19,7 +34,6 @@ async function fetchHandleError(url) { rsp = await fetch(url).then(function (response) { if (!response.ok) { // make the promise be rejected if we didn't get a 2xx response - console.log("response.statusText", response.statusText, 'statusCode', response.status) // NB. statusText could be "Not Found" or "File not found" depending on server // Standardise based on response.status if (response.status == 404) { @@ -101,24 +115,26 @@ export async function getText(url) { let schemas = {}; -export async function getSchema(version, schemaName = "image") { - let cacheKey = schemaName + version; - if (!schemas[cacheKey]) { - const schema_url = getSchemaUrl(schemaName, version); - console.log("Loading schema... " + schema_url); - try { - const schema = await getJson(schema_url); - // delete to avoid invalid: $schema: "https://json-schema.org/draft/2020-12/schema" not found - delete schema["$schema"]; - schemas[cacheKey] = schema; - } catch (error) { - throw new Error(`No schema at ${schema_url}. Version ${version} may be invalid.`); - } +export async function getSchema(schemaUrl) { + if (!schemas[schemaUrl]) { + console.log("Loading schema... " + schemaUrl); + const schema = await getJson(schemaUrl); + // delete to avoid invalid: $schema: "https://json-schema.org/draft/2020-12/schema" not found + delete schema["$schema"]; + schemas[schemaUrl] = schema; } - return schemas[cacheKey]; + return schemas[schemaUrl]; } export function getVersion(jsonData) { + if (jsonData.attributes) { + for(let v=0; v getSchemaUrl(name, version)); } @@ -184,22 +204,23 @@ export function validateData(schema, jsonData) { export async function validate(jsonData) { // get version, lookup schema, do validation... - const schemaNames = getSchemaNames(jsonData); + let version = getVersion(jsonData); + console.log("VERSION", version); + + const schemaUrls = getSchemaUrlsForJson(jsonData); - if (schemaNames.length == 0) { + if (schemaUrls.length == 0) { return ["Unrecognised JSON data"]; } - let version = getVersion(jsonData); - if (!version) { console.log("No version found, using: " + CURRENT_VERSION); version = CURRENT_VERSION; } let errors = []; - for (let s=0; s Date: Wed, 12 Jun 2024 13:12:08 +0100 Subject: [PATCH 03/33] Fix display of Array info for Zarr v3 --- .../MultiscaleArrays/Multiscale.svelte | 10 ++-- .../ZarrArray/ChunkLoader.svelte | 4 +- .../MultiscaleArrays/ZarrArray/Cube3D.svelte | 3 +- .../MultiscaleArrays/ZarrArray/index.svelte | 16 +++--- src/JsonValidator/index.svelte | 17 ++++--- src/utils.js | 50 +++++++++++++------ 6 files changed, 64 insertions(+), 36 deletions(-) diff --git a/src/JsonValidator/MultiscaleArrays/Multiscale.svelte b/src/JsonValidator/MultiscaleArrays/Multiscale.svelte index 7ddbeef..97e5130 100644 --- a/src/JsonValidator/MultiscaleArrays/Multiscale.svelte +++ b/src/JsonValidator/MultiscaleArrays/Multiscale.svelte @@ -1,5 +1,5 @@
-

Path {path + "/.zarray"}

+

Path {path}

{#await promise}
loading array .zarray ...
@@ -64,7 +64,7 @@
- {path}/.zarray + {path}
{JSON.stringify(zarray, null, 2)}
{:catch error} diff --git a/src/JsonValidator/MultiscaleArrays/index.svelte b/src/JsonValidator/MultiscaleArrays/index.svelte index 1ba56cf..d654097 100644 --- a/src/JsonValidator/MultiscaleArrays/index.svelte +++ b/src/JsonValidator/MultiscaleArrays/index.svelte @@ -1,9 +1,13 @@ {#each rootAttrs.multiscales as multiscale, idx} @@ -11,7 +15,7 @@

Multiscale {idx}

{#each multiscale.datasets as dataset} - + {/each} {/each} diff --git a/src/JsonValidator/index.svelte b/src/JsonValidator/index.svelte index 87f8222..97a4514 100644 --- a/src/JsonValidator/index.svelte +++ b/src/JsonValidator/index.svelte @@ -15,6 +15,7 @@ getJson, getVersion, getDataType, + getZarrGroupAttrsFileName, } from "../utils"; export let source; @@ -32,12 +33,13 @@ const zarrName = dirs[dirs.length - 1]; // check for labels/.zattrs - const labelsPromise = getJson(source + '/labels/.zattrs'); + const zarrAttrsFileName = getZarrGroupAttrsFileName(msVersion); + const labelsPromise = getJson(source + '/labels/' + zarrAttrsFileName);

- Validating: /{zarrName}/.zattrs + Validating: /{zarrName}/{zarrAttrsFileName}

{#if !msVersion}No version found. Using {CURRENT_VERSION}
{/if} diff --git a/src/utils.js b/src/utils.js index 5a162bd..b5a9329 100644 --- a/src/utils.js +++ b/src/utils.js @@ -66,6 +66,19 @@ async function fetchHandleError(url) { throw Error(msg); } +export function getZarrGroupAttrsFileName(ngffVersion) { + if (["0.1", "0.2", "0.3", "0.4"].includes(ngffVersion)) { + return ".zattrs"; + } + return "zarr.json"; +} + +export function getZarrArrayAttrsFileName(ngffVersion) { + if (["0.1", "0.2", "0.3", "0.4"].includes(ngffVersion)) { + return ".zarray"; + } + return "zarr.json"; +} export async function getZarrArrayJson(zarr_dir) { return getZarrJson(zarr_dir, ".zarray"); @@ -145,6 +158,7 @@ export function getNgffData(jsonData) { export function getVersion(jsonData) { let ngffData = getNgffData(jsonData); + // TODO: v0.5 won't likely have version at multiscales[0].version console.log("getVersion", jsonData, ngffData); let version = ngffData.multiscales ? ngffData.multiscales[0].version From 6940281bb1df7da4f2405dd19635d8d7e8366f0f Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 13 Jun 2024 21:59:27 +0100 Subject: [PATCH 05/33] Update loading of zarr chunks with zarrita --- package-lock.json | 248 +++++++++++++----- package.json | 3 +- .../ZarrArray/ChunkLoader.svelte | 32 ++- .../ZarrArray/ChunkViewer.svelte | 26 +- vite.config.js | 12 +- 5 files changed, 236 insertions(+), 85 deletions(-) diff --git a/package-lock.json b/package-lock.json index 981019f..3291474 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,9 +9,10 @@ "version": "0.3.0", "dependencies": { "ajv": "^8.11.0", + "ndarray": "^1.0.19", "svelte-icons-pack": "^1.4.6", "svelte-simple-modal": "^1.4.1", - "zarr": "^0.6.0" + "zarrita": "^0.4.0-next.13" }, "devDependencies": { "@sveltejs/vite-plugin-svelte": "^1.0.0-next.30", @@ -59,6 +60,40 @@ } } }, + "node_modules/@zarrita/core": { + "version": "0.1.0-next.11", + "resolved": "https://registry.npmjs.org/@zarrita/core/-/core-0.1.0-next.11.tgz", + "integrity": "sha512-x2sjxKD8R9O+Bu7ldoRDsOSFoHzWT/owG1qjUnB3I4AQ0/aHay4rDU1ePXz27RZRmlZX32s9GCQh7UtX4nRXtg==", + "dependencies": { + "@zarrita/storage": "^0.1.0-next.5", + "@zarrita/typedarray": "^0.1.0-next.3", + "numcodecs": "^0.3.1" + } + }, + "node_modules/@zarrita/indexing": { + "version": "0.1.0-next.13", + "resolved": "https://registry.npmjs.org/@zarrita/indexing/-/indexing-0.1.0-next.13.tgz", + "integrity": "sha512-qxBIUHpBRLChc71uY8+GkQKaRip1sgFGA1STnSxDee6gMuqEaB0f1SBoa+VPf8eMK6mLEatTE9KXm6pRzRYj/A==", + "dependencies": { + "@zarrita/core": "^0.1.0-next.11", + "@zarrita/storage": "^0.1.0-next.5", + "@zarrita/typedarray": "^0.1.0-next.3" + } + }, + "node_modules/@zarrita/storage": { + "version": "0.1.0-next.5", + "resolved": "https://registry.npmjs.org/@zarrita/storage/-/storage-0.1.0-next.5.tgz", + "integrity": "sha512-E1VSxhNGZHL4RsKfIuyaz0HRsDk7hOU8Y7R+8yvKolaHDjK31XQsUgu97oaR24qS1j1OOg5vGyFyd+y0q7FNOA==", + "dependencies": { + "reference-spec-reader": "^0.2.0", + "unzipit": "^1.4.3" + } + }, + "node_modules/@zarrita/typedarray": { + "version": "0.1.0-next.3", + "resolved": "https://registry.npmjs.org/@zarrita/typedarray/-/typedarray-0.1.0-next.3.tgz", + "integrity": "sha512-DpSaU3Cr6HmYDC/v8oM+e219cHU/kzKma309Z9E+QbpRnZycKNbSTKcxFR7FqB6HgB9640gzNUVFG5P+wzX5Xg==" + }, "node_modules/ajv": { "version": "8.11.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", @@ -461,16 +496,16 @@ "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", "dev": true }, - "node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" - }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==" + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -503,6 +538,16 @@ "node": ">= 0.4.0" } }, + "node_modules/iota-array": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz", + "integrity": "sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==" + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "node_modules/is-core-module": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", @@ -559,38 +604,21 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/numcodecs": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.2.2.tgz", - "integrity": "sha512-Y5K8mv80yb4MgVpcElBkUeMZqeE4TrovxRit/dTZvoRl6YkB6WEjY+fiUjGCblITnt3T3fmrDg8yRWu0gOLjhQ==", - "engines": { - "node": ">=12" - } - }, - "node_modules/p-queue": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-7.3.0.tgz", - "integrity": "sha512-5fP+yVQ0qp0rEfZoDTlP2c3RYBgxvRsw30qO+VtPPc95lyvSG+x6USSh1TuLB4n96IO6I8/oXQGsTgtna4q2nQ==", + "node_modules/ndarray": { + "version": "1.0.19", + "resolved": "https://registry.npmjs.org/ndarray/-/ndarray-1.0.19.tgz", + "integrity": "sha512-B4JHA4vdyZU30ELBw3g7/p9bZupyew5a7tX1Y/gGeF2hafrPaQZhgrGQfsvgfYbgdFZjYwuEcnaobeM/WMW+HQ==", "dependencies": { - "eventemitter3": "^4.0.7", - "p-timeout": "^5.0.2" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "iota-array": "^1.0.0", + "is-buffer": "^1.0.2" } }, - "node_modules/p-timeout": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-5.1.0.tgz", - "integrity": "sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node_modules/numcodecs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.3.1.tgz", + "integrity": "sha512-ywIyGpJ+c6Ojktq9a8jsWSy12ZSUcW/W+I3jlH0q0zv9aR/ZiMsN7IrWaNq9YV2FRdLu6r/M6lp35jMA6fug/A==", + "dependencies": { + "fflate": "^0.8.0" } }, "node_modules/path-parse": { @@ -649,6 +677,11 @@ "node": ">=6" } }, + "node_modules/reference-spec-reader": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/reference-spec-reader/-/reference-spec-reader-0.2.0.tgz", + "integrity": "sha512-q0mfCi5yZSSHXpCyxjgQeaORq3tvDsxDyzaadA/5+AbAUwRyRuuTh0aRQuE/vAOt/qzzxidJ5iDeu1cLHaNBlQ==" + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -749,6 +782,17 @@ "svelte": "^3.31.2" } }, + "node_modules/unzipit": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unzipit/-/unzipit-1.4.3.tgz", + "integrity": "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg==", + "dependencies": { + "uzip-module": "^1.0.2" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -757,6 +801,11 @@ "punycode": "^2.1.0" } }, + "node_modules/uzip-module": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/uzip-module/-/uzip-module-1.0.3.tgz", + "integrity": "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA==" + }, "node_modules/vite": { "version": "2.9.9", "resolved": "https://registry.npmjs.org/vite/-/vite-2.9.9.tgz", @@ -794,16 +843,14 @@ } } }, - "node_modules/zarr": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/zarr/-/zarr-0.6.0.tgz", - "integrity": "sha512-WgGe+aFyK6EmTyZcqJ2OFeFQ9fJFuJqeJgM5sKnbap7HJboaI+rBpDp0rPXSTq5ITu/rLEQjZQ2mPlcfYk7DYA==", + "node_modules/zarrita": { + "version": "0.4.0-next.13", + "resolved": "https://registry.npmjs.org/zarrita/-/zarrita-0.4.0-next.13.tgz", + "integrity": "sha512-F0ujRsh5G8uuNe1CH12I9o/Q88Jc3YnRp4s7QzOIjvXvXzEv8Ht2VLCbZsTiS/pI4ynxKr1+9XS1LZixhoGDVA==", "dependencies": { - "numcodecs": "^0.2.2", - "p-queue": "^7.1.0" - }, - "engines": { - "node": ">=12" + "@zarrita/core": "^0.1.0-next.11", + "@zarrita/indexing": "^0.1.0-next.13", + "@zarrita/storage": "^0.1.0-next.5" } } }, @@ -832,6 +879,40 @@ "svelte-hmr": "^0.14.11" } }, + "@zarrita/core": { + "version": "0.1.0-next.11", + "resolved": "https://registry.npmjs.org/@zarrita/core/-/core-0.1.0-next.11.tgz", + "integrity": "sha512-x2sjxKD8R9O+Bu7ldoRDsOSFoHzWT/owG1qjUnB3I4AQ0/aHay4rDU1ePXz27RZRmlZX32s9GCQh7UtX4nRXtg==", + "requires": { + "@zarrita/storage": "^0.1.0-next.5", + "@zarrita/typedarray": "^0.1.0-next.3", + "numcodecs": "^0.3.1" + } + }, + "@zarrita/indexing": { + "version": "0.1.0-next.13", + "resolved": "https://registry.npmjs.org/@zarrita/indexing/-/indexing-0.1.0-next.13.tgz", + "integrity": "sha512-qxBIUHpBRLChc71uY8+GkQKaRip1sgFGA1STnSxDee6gMuqEaB0f1SBoa+VPf8eMK6mLEatTE9KXm6pRzRYj/A==", + "requires": { + "@zarrita/core": "^0.1.0-next.11", + "@zarrita/storage": "^0.1.0-next.5", + "@zarrita/typedarray": "^0.1.0-next.3" + } + }, + "@zarrita/storage": { + "version": "0.1.0-next.5", + "resolved": "https://registry.npmjs.org/@zarrita/storage/-/storage-0.1.0-next.5.tgz", + "integrity": "sha512-E1VSxhNGZHL4RsKfIuyaz0HRsDk7hOU8Y7R+8yvKolaHDjK31XQsUgu97oaR24qS1j1OOg5vGyFyd+y0q7FNOA==", + "requires": { + "reference-spec-reader": "^0.2.0", + "unzipit": "^1.4.3" + } + }, + "@zarrita/typedarray": { + "version": "0.1.0-next.3", + "resolved": "https://registry.npmjs.org/@zarrita/typedarray/-/typedarray-0.1.0-next.3.tgz", + "integrity": "sha512-DpSaU3Cr6HmYDC/v8oM+e219cHU/kzKma309Z9E+QbpRnZycKNbSTKcxFR7FqB6HgB9640gzNUVFG5P+wzX5Xg==" + }, "ajv": { "version": "8.11.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", @@ -1032,16 +1113,16 @@ "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", "dev": true }, - "eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" - }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, + "fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==" + }, "fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -1064,6 +1145,16 @@ "function-bind": "^1.1.1" } }, + "iota-array": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz", + "integrity": "sha512-pZ2xT+LOHckCatGQ3DcG/a+QuEqvoxqkiL7tvE8nn3uuu+f6i1TtpB5/FtWFbxUuVr5PZCx8KskuGatbJDXOWA==" + }, + "is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "is-core-module": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.9.0.tgz", @@ -1105,24 +1196,22 @@ "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", "dev": true }, - "numcodecs": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.2.2.tgz", - "integrity": "sha512-Y5K8mv80yb4MgVpcElBkUeMZqeE4TrovxRit/dTZvoRl6YkB6WEjY+fiUjGCblITnt3T3fmrDg8yRWu0gOLjhQ==" - }, - "p-queue": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-7.3.0.tgz", - "integrity": "sha512-5fP+yVQ0qp0rEfZoDTlP2c3RYBgxvRsw30qO+VtPPc95lyvSG+x6USSh1TuLB4n96IO6I8/oXQGsTgtna4q2nQ==", + "ndarray": { + "version": "1.0.19", + "resolved": "https://registry.npmjs.org/ndarray/-/ndarray-1.0.19.tgz", + "integrity": "sha512-B4JHA4vdyZU30ELBw3g7/p9bZupyew5a7tX1Y/gGeF2hafrPaQZhgrGQfsvgfYbgdFZjYwuEcnaobeM/WMW+HQ==", "requires": { - "eventemitter3": "^4.0.7", - "p-timeout": "^5.0.2" + "iota-array": "^1.0.0", + "is-buffer": "^1.0.2" } }, - "p-timeout": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-5.1.0.tgz", - "integrity": "sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==" + "numcodecs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/numcodecs/-/numcodecs-0.3.1.tgz", + "integrity": "sha512-ywIyGpJ+c6Ojktq9a8jsWSy12ZSUcW/W+I3jlH0q0zv9aR/ZiMsN7IrWaNq9YV2FRdLu6r/M6lp35jMA6fug/A==", + "requires": { + "fflate": "^0.8.0" + } }, "path-parse": { "version": "1.0.7", @@ -1158,6 +1247,11 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, + "reference-spec-reader": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/reference-spec-reader/-/reference-spec-reader-0.2.0.tgz", + "integrity": "sha512-q0mfCi5yZSSHXpCyxjgQeaORq3tvDsxDyzaadA/5+AbAUwRyRuuTh0aRQuE/vAOt/qzzxidJ5iDeu1cLHaNBlQ==" + }, "require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -1224,6 +1318,14 @@ "integrity": "sha512-em/uxH1xvQZoXTOq81Kk0u9ltjf/EyQkNiKTQJQmdCygDMqyUfMCFzLnbIQ4ApfV4BcRh6eYbwbCeeWTOyfpsg==", "requires": {} }, + "unzipit": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unzipit/-/unzipit-1.4.3.tgz", + "integrity": "sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg==", + "requires": { + "uzip-module": "^1.0.2" + } + }, "uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -1232,6 +1334,11 @@ "punycode": "^2.1.0" } }, + "uzip-module": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/uzip-module/-/uzip-module-1.0.3.tgz", + "integrity": "sha512-AMqwWZaknLM77G+VPYNZLEruMGWGzyigPK3/Whg99B3S6vGHuqsyl5ZrOv1UUF3paGK1U6PM0cnayioaryg/fA==" + }, "vite": { "version": "2.9.9", "resolved": "https://registry.npmjs.org/vite/-/vite-2.9.9.tgz", @@ -1245,13 +1352,14 @@ "rollup": "^2.59.0" } }, - "zarr": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/zarr/-/zarr-0.6.0.tgz", - "integrity": "sha512-WgGe+aFyK6EmTyZcqJ2OFeFQ9fJFuJqeJgM5sKnbap7HJboaI+rBpDp0rPXSTq5ITu/rLEQjZQ2mPlcfYk7DYA==", + "zarrita": { + "version": "0.4.0-next.13", + "resolved": "https://registry.npmjs.org/zarrita/-/zarrita-0.4.0-next.13.tgz", + "integrity": "sha512-F0ujRsh5G8uuNe1CH12I9o/Q88Jc3YnRp4s7QzOIjvXvXzEv8Ht2VLCbZsTiS/pI4ynxKr1+9XS1LZixhoGDVA==", "requires": { - "numcodecs": "^0.2.2", - "p-queue": "^7.1.0" + "@zarrita/core": "^0.1.0-next.11", + "@zarrita/indexing": "^0.1.0-next.13", + "@zarrita/storage": "^0.1.0-next.5" } } } diff --git a/package.json b/package.json index f66b677..8831a69 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,9 @@ }, "dependencies": { "ajv": "^8.11.0", + "ndarray": "^1.0.19", "svelte-icons-pack": "^1.4.6", "svelte-simple-modal": "^1.4.1", - "zarr": "^0.6.0" + "zarrita": "^0.4.0-next.13" } } diff --git a/src/JsonValidator/MultiscaleArrays/ZarrArray/ChunkLoader.svelte b/src/JsonValidator/MultiscaleArrays/ZarrArray/ChunkLoader.svelte index 3834b96..5735e86 100644 --- a/src/JsonValidator/MultiscaleArrays/ZarrArray/ChunkLoader.svelte +++ b/src/JsonValidator/MultiscaleArrays/ZarrArray/ChunkLoader.svelte @@ -1,13 +1,19 @@ diff --git a/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte b/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte index 7206b97..9634667 100644 --- a/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte +++ b/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte @@ -31,7 +31,7 @@

Path {path}

{#await promise} -
loading array .zarray ...
+
loading array data ...
{:then zarray} diff --git a/src/JsonValidator/index.svelte b/src/JsonValidator/index.svelte index 97a4514..75b6208 100644 --- a/src/JsonValidator/index.svelte +++ b/src/JsonValidator/index.svelte @@ -11,7 +11,6 @@ CURRENT_VERSION, getSchemaUrlsForJson, validate, - getNgffData, getJson, getVersion, getDataType, @@ -24,7 +23,6 @@ const msVersion = getVersion(rootAttrs); const dtype = getDataType(rootAttrs); - const ngffData = getNgffData(rootAttrs); const schemaUrls = getSchemaUrlsForJson(rootAttrs); console.log("index.svelte schemaUrls", schemaUrls) const promise = validate(rootAttrs); @@ -82,12 +80,12 @@ {/await} -{#if ngffData.multiscales} - -{:else if ngffData.plate} - -{:else if ngffData.well} - +{#if rootAttrs.multiscales} + +{:else if rootAttrs.plate} + +{:else if rootAttrs.well} + {/if} \ No newline at end of file diff --git a/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte b/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte index 7e9cb45..379ac15 100644 --- a/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte +++ b/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte @@ -2,6 +2,7 @@ import { getJson, formatBytes, getChunkAndShardShapes, getArrayDtype } from "../../../utils"; import Cube3D from "./Cube3D.svelte"; import ChunkLoader from "./ChunkLoader.svelte"; + import DetailsPrePanel from "../../../JsonBrowser/DetailsPrePanel.svelte"; export let source; export let path; @@ -82,10 +83,9 @@ -
- {path} -
{JSON.stringify(zarray, null, 2)}
-
+
+ +
{:catch error}

{error.message}

{/await} @@ -101,13 +101,6 @@ text-align: center; } - pre { - color: #faebd7; - background-color: #2c3e50; - padding: 10px; - font-size: 14px; - } - table { background-color: white; font-size: 14px; @@ -133,18 +126,4 @@ a:visited { color: #ff512f; } - - details { - font-size: 1.1em; - margin: 0 15px; - text-align: left; - } - pre { - margin-top: 10px; - color: #faebd7; - background-color: #2c3e50; - padding: 10px; - font-size: 14px; - border-radius: 10px; - } diff --git a/src/JsonValidator/RoCrate/index.svelte b/src/JsonValidator/RoCrate/index.svelte new file mode 100644 index 0000000..9a19696 --- /dev/null +++ b/src/JsonValidator/RoCrate/index.svelte @@ -0,0 +1,18 @@ + + +{#await promise} +

Loading ro-crate-metadata.json...

+{:then jsonData} + {#if jsonData} + + {/if} +{:catch error} +

No ro-crate-metadata.json found

+{/await} diff --git a/src/JsonValidator/index.svelte b/src/JsonValidator/index.svelte index 52e008a..75e9778 100644 --- a/src/JsonValidator/index.svelte +++ b/src/JsonValidator/index.svelte @@ -1,6 +1,7 @@ + + +
    + {#each report as {name, value}} +
  • + {name}: + {#if value} + + {#if value.startsWith("http")} + {value} + {:else} + {value} + {/if} + {:else} + x Not found + {/if} +
  • + {/each} +
+ + + diff --git a/src/JsonValidator/RoCrate/index.svelte b/src/JsonValidator/RoCrate/index.svelte index 9a19696..3afeeac 100644 --- a/src/JsonValidator/RoCrate/index.svelte +++ b/src/JsonValidator/RoCrate/index.svelte @@ -1,18 +1,40 @@ + +
+

Ro-crate metadata

+ + + {#await promise}

Loading ro-crate-metadata.json...

{:then jsonData} {#if jsonData} - + + {/if} {:catch error}

No ro-crate-metadata.json found

{/await} + +
+ + + \ No newline at end of file diff --git a/src/JsonValidator/index.svelte b/src/JsonValidator/index.svelte index 75e9778..f8f50dd 100644 --- a/src/JsonValidator/index.svelte +++ b/src/JsonValidator/index.svelte @@ -75,7 +75,10 @@ - + + {#if !["0.1", "0.2", "0.3", "0.4"].includes(msVersion)} + + {/if} {#await labelsPromise}

checking for labels...

From b81651459107ee69950db72c8d585f87613f8c1c Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 22 Aug 2024 08:42:29 +0100 Subject: [PATCH 22/33] Handle data with no sharding codecs --- src/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index dc0a613..5692f66 100644 --- a/src/utils.js +++ b/src/utils.js @@ -276,7 +276,7 @@ export function getChunkAndShardShapes(zarray) { // Based on https://github.com/zarr-developers/zarr-specs/blob/main/docs/v3/codecs/sharding-indexed/v1.0.rst#configuration-parameters const chunk_shape = zarray.chunk_grid?.configuration?.chunk_shape; let sharding_codecs = zarray.codecs?.filter(codec => codec.name == "sharding_indexed"); - const sub_chunks = sharding_codecs?.[0].configuration?.chunk_shape; + const sub_chunks = sharding_codecs?.[0]?.configuration?.chunk_shape; // if we have sharding, a 'chunk' is the sub-chunk of a shard if (sub_chunks) { return [sub_chunks, chunk_shape] From 0256c077599c212818451b5898d3ced1bfa4de44 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 22 Aug 2024 12:24:30 +0100 Subject: [PATCH 23/33] Try to parse and lookup Organism and Imaging method from Ro-crate --- .../RoCrate/RoCrateValidator.svelte | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/JsonValidator/RoCrate/RoCrateValidator.svelte b/src/JsonValidator/RoCrate/RoCrateValidator.svelte index 7c2a60e..f35ba4e 100644 --- a/src/JsonValidator/RoCrate/RoCrateValidator.svelte +++ b/src/JsonValidator/RoCrate/RoCrateValidator.svelte @@ -1,11 +1,44 @@
    - {#each report as {name, value}} + {#each report as {name, value, level}}
  • {name}: {#if value} - + {@html warningSymbols.OK } {#if value.startsWith("http")} {value} {:else} {value} {/if} {:else} - x Not found + {@html warningSymbols[level] } Not found {/if}
  • {/each}
  • Organism: {#if organismId} - {organismId} {organismName} + {@html warningSymbols.OK } {organismId} {organismName} {:else} - x Not found + {@html warningSymbols.SUGGESTED } Not found {/if}
  • Imaging method: {#if fbbiId} - {fbbiId} {imagingMethod} + {@html warningSymbols.OK } {fbbiId} {imagingMethod} {:else} - x Not found + {@html warningSymbols.SUGGESTED } Not found {/if}
From e09b012c2ba7ecf82e41a4cecc807dfc5ae4efd2 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 2 Sep 2024 16:54:43 +0100 Subject: [PATCH 25/33] Improve version checking. Handle missing version 0.4 --- .../RoCrate/RoCrateValidator.svelte | 5 +++ src/JsonValidator/index.svelte | 32 ++++++++++++++--- src/utils.js | 36 +++++++++++++++---- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/src/JsonValidator/RoCrate/RoCrateValidator.svelte b/src/JsonValidator/RoCrate/RoCrateValidator.svelte index edb74f2..a8035fd 100644 --- a/src/JsonValidator/RoCrate/RoCrateValidator.svelte +++ b/src/JsonValidator/RoCrate/RoCrateValidator.svelte @@ -103,4 +103,9 @@ margin-bottom: 10px; } + .suggested { + color: orange; + font-size: 20px; + } + diff --git a/src/JsonValidator/index.svelte b/src/JsonValidator/index.svelte index f8f50dd..2a18444 100644 --- a/src/JsonValidator/index.svelte +++ b/src/JsonValidator/index.svelte @@ -21,10 +21,32 @@ export let source; export let rootAttrs; + let versionMessage= ""; + let version = null; + // Initial validation of version + // Top level 'version' is required "MUST" for v0.5+ + if (rootAttrs?.attributes?.ome) { + if (!rootAttrs.attributes.ome.version) { + versionMessage = "Missing 'version' field under attributes.ome."; + } else { + version = rootAttrs.attributes.ome.version; + versionMessage = `Using version ${version}.`; + } + } else { + // version for older versions can be nested under "multiscales" or "plate" or "well" + version = getVersion(rootAttrs); + if (!version) { + versionMessage = "No version found. Using version 0.4."; + version = "0.4"; + } else { + versionMessage = `Using version ${version}.`; + } + } + + // v0.5+ unwrap the attrs under "ome" const omeAttrs = rootAttrs?.attributes?.ome || rootAttrs; - const msVersion = getVersion(rootAttrs); const dtype = getDataType(omeAttrs); const schemaUrls = getSchemaUrlsForJson(omeAttrs); @@ -35,7 +57,7 @@ const zarrName = dirs[dirs.length - 1]; // check for labels/.zattrs - const zarrAttrsFileName = getZarrGroupAttrsFileName(msVersion); + const zarrAttrsFileName = getZarrGroupAttrsFileName(version); const labelsPromise = getJson(source + '/labels/' + zarrAttrsFileName); @@ -44,7 +66,7 @@ Validating: /{zarrName}/{zarrAttrsFileName}

- {#if !msVersion}No version found. Using {CURRENT_VERSION}
{/if} + {versionMessage} Using schema{schemaUrls.length > 1 ? "s" : ""}: {#each schemaUrls as url, i} @@ -72,11 +94,11 @@
- +
- {#if !["0.1", "0.2", "0.3", "0.4"].includes(msVersion)} + {#if !["0.1", "0.2", "0.3", "0.4"].includes(version)} {/if} diff --git a/src/utils.js b/src/utils.js index 5692f66..5ae2f61 100644 --- a/src/utils.js +++ b/src/utils.js @@ -140,13 +140,26 @@ export async function getSchema(schemaUrl) { } export function getVersion(ngffData) { - console.log("getVersion...", ngffData, ngffData.ome?.version) + console.log("getVersion...", ngffData, 'attributes?', ngffData.attributes) + // if we have attributes.ome then this is version 0.5+ + if (ngffData.attributes?.ome) { + if (ngffData.attributes.ome.version) { + return ngffData.attributes.ome.version; + } else { + throw Error("No version found in attributes.ome"); + } + } + + // Used if we have our 'attributes' at the root if (ngffData.ome?.version) { + console.trace("WARNING - using ngffData.ome?.version FIXME?") return ngffData.ome.version; } if (ngffData.version) { + console.trace("WARNING - using ngffData.version FIXME?") return ngffData.version; } + // Handle version 0.4 and earlier let version = ngffData.multiscales ? ngffData.multiscales[0].version : ngffData.plate @@ -155,7 +168,10 @@ export function getVersion(ngffData) { ? ngffData.well.version : undefined; console.log("version", version); - return version; + // for 0.4 and earlier, version wasn't MUST and we defaulted + // to using v0.4 for validation. To preserve that behaviour + // return "0.4" if no version found. + return version || "0.4"; } export function toTitleCase(text) { @@ -200,11 +216,6 @@ export function getSchemaUrlsForJson(rootAttrs) { const msVersion = getVersion(omeAttrs); const version = msVersion || CURRENT_VERSION; - // for v0.5 onwards, rootAttrs is nested under attributes.ome... - if (omeAttrs.ome) { - console.trace("WARNING - FIXME!") - omeAttrs = omeAttrs.ome; - } const schemaNames = getSchemaNames(omeAttrs); return schemaNames.map(name => getSchemaUrl(name, version)); } @@ -227,7 +238,18 @@ export async function validate(jsonData) { // get version, lookup schema, do validation... // v0.5+ unwrap the attrs under "attributes.ome" let omeAttrs = jsonData?.attributes?.ome || jsonData; + let version = getVersion(omeAttrs); + // v0.5+ (with attributes.ome) MUST have top-level version + if (jsonData?.attributes?.ome) { + if (!jsonData.attributes.ome.version) { + return ["No version found under attributes.ome"]; + } + } else if (!version) { + // default to last version pre 0.5 rules. + version = "0.4"; + } + console.log("validate VERSION", version, jsonData); const schemaUrls = getSchemaUrlsForJson(jsonData); From d2dfb1f52f10fe17b1341f68cde51dcdba135f28 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 3 Sep 2024 11:07:39 +0100 Subject: [PATCH 26/33] Fix bioformats2raw page with v0.5 data --- src/App.svelte | 9 ++++++--- src/Bioformats2rawLayout/index.svelte | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/App.svelte b/src/App.svelte index faafd2a..2be7a3d 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -15,8 +15,6 @@ source = source.slice(0, -1); } - let location = window.location.href; - let promise; if (source) { @@ -24,6 +22,11 @@ console.log("Loading JSON... " + source); promise = getZarrGroupAttrs(source); } + + function isBioFormats2Raw(data) { + let omeAttrs = data?.attributes?.ome || data; + return omeAttrs["bioformats2raw.layout"] === 3 && !omeAttrs.plate; + } @@ -38,7 +41,7 @@ {:then data} <div> - {#if data["bioformats2raw.layout"] === 3 && !data.plate} + {#if isBioFormats2Raw(data)} <Bioformats2rawLayout rootAttrs={data} {source} /> {:else} <JsonValidator rootAttrs={data} {source} /> diff --git a/src/Bioformats2rawLayout/index.svelte b/src/Bioformats2rawLayout/index.svelte index c31967c..2be48b5 100644 --- a/src/Bioformats2rawLayout/index.svelte +++ b/src/Bioformats2rawLayout/index.svelte @@ -1,5 +1,5 @@ <script> - import { getXmlDom, getJson, validate } from "../utils"; + import { getXmlDom, getZarrGroupAttrs, validate } from "../utils"; import JsonBrowser from "../JsonBrowser/index.svelte"; import ImageContainer from "../JsonValidator/Well/ImageContainer.svelte"; @@ -40,7 +40,7 @@ // wait for schema to be cached, so we don't load them multiple times // let schemasPromise = getSchema("0.2", "image"); async function preloadSchema(imagePath) { - let imgAttrs = await getJson(imagePath + "/.zattrs"); + let imgAttrs = getZarrGroupAttrs(imagePath); console.log("preloadSchema", imgAttrs); let errs = await validate(imgAttrs); return errs; From 0efe2a22b087c36d5e75188c86acf93b6a547f6a Mon Sep 17 00:00:00 2001 From: William Moore <w.moore@dundee.ac.uk> Date: Tue, 3 Sep 2024 11:30:45 +0100 Subject: [PATCH 27/33] Fix validation of Wells in Plate view --- src/JsonValidator/Plate/WellContainer/PlateWell.svelte | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/JsonValidator/Plate/WellContainer/PlateWell.svelte b/src/JsonValidator/Plate/WellContainer/PlateWell.svelte index e0a7d94..4822f5e 100644 --- a/src/JsonValidator/Plate/WellContainer/PlateWell.svelte +++ b/src/JsonValidator/Plate/WellContainer/PlateWell.svelte @@ -5,6 +5,10 @@ export let source; export let path; + let rootAttrs = wellAttrs; + // unwrap attributes.ome if present + wellAttrs = wellAttrs.attributes?.ome || wellAttrs; + const wellToValidate = getSearchParam("well"); let wellIndex = parseInt(wellToValidate); let wellIndices = [0]; @@ -22,7 +26,7 @@ let errs = []; console.log("loadAndValidate wellAttrs", wellAttrs); - const imgs = wellAttrs.attributes?.ome?.well.images || wellAttrs.well.images; + const imgs = wellAttrs.well.images; for (let i=0; i < wellIndices.length; i++) { // this will fail if e.g. any getZarrGroupAttrs() raises exception let index = wellIndices[i]; @@ -37,7 +41,7 @@ return errs; } - let validatePromise = validate(wellAttrs); + let validatePromise = validate(rootAttrs); let imagePromise = loadAndValidate(); From 1d55e986d8b91ba522bca5b3d58dc3bfeab0ac11 Mon Sep 17 00:00:00 2001 From: William Moore <w.moore@dundee.ac.uk> Date: Tue, 24 Sep 2024 15:28:55 +0100 Subject: [PATCH 28/33] Add ro-crate-metadata support to bioformats2raw component --- src/Bioformats2rawLayout/index.svelte | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/Bioformats2rawLayout/index.svelte b/src/Bioformats2rawLayout/index.svelte index 2be48b5..5575722 100644 --- a/src/Bioformats2rawLayout/index.svelte +++ b/src/Bioformats2rawLayout/index.svelte @@ -1,13 +1,19 @@ <script> - import { getXmlDom, getZarrGroupAttrs, validate } from "../utils"; + import { getXmlDom, getZarrGroupAttrs, validate, getVersion, getZarrGroupAttrsFileName } from "../utils"; import JsonBrowser from "../JsonBrowser/index.svelte"; import ImageContainer from "../JsonValidator/Well/ImageContainer.svelte"; + import RoCrate from "../JsonValidator/RoCrate/index.svelte"; export let source; export let rootAttrs; const metadataName = "OME/METADATA.ome.xml"; + const version = getVersion(rootAttrs); + console.log("BF2RAW version", version, rootAttrs); + const zarrAttrsFileName = getZarrGroupAttrsFileName(version); + console.log("zarrAttrsFileName", zarrAttrsFileName); + // source/OME/METADATA.ome.xml const metadataUrl = `${source}/${metadataName}`; @@ -53,7 +59,7 @@ </script> <article> - Reading: <a href={source}>/{zarrName}/.zattrs</a> + Reading: <a href={source}>/{zarrName}/{zarrAttrsFileName}</a> <div class="json"> <JsonBrowser name="" version="" contents={rootAttrs} expanded /> @@ -99,6 +105,11 @@ {:catch error} <p style="color: red">{error.message}</p> {/await} + + <!-- for v0.5+ we check for ro-crate-metadata.json --> + {#if !["0.1", "0.2", "0.3", "0.4"].includes(version)} + <RoCrate {source}></RoCrate> + {/if} </article> <style> From c0374f59dba1f9c776b2c6847abd55b2d3358e16 Mon Sep 17 00:00:00 2001 From: William Moore <w.moore@dundee.ac.uk> Date: Fri, 27 Sep 2024 11:14:54 +0100 Subject: [PATCH 29/33] Load _version schema. Validate using 'attributes' as JSON root --- src/utils.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/utils.js b/src/utils.js index 5ae2f61..99c6904 100644 --- a/src/utils.js +++ b/src/utils.js @@ -140,7 +140,7 @@ export async function getSchema(schemaUrl) { } export function getVersion(ngffData) { - console.log("getVersion...", ngffData, 'attributes?', ngffData.attributes) + // console.log("getVersion...", ngffData, 'attributes?', ngffData.attributes) // if we have attributes.ome then this is version 0.5+ if (ngffData.attributes?.ome) { if (ngffData.attributes.ome.version) { @@ -267,16 +267,22 @@ export async function validate(jsonData) { // TODO: need to know whether to load other schemas... // For now, we can use version check... if (version === "0.5") { - // const ctSchema = await getSchema(version, "coordinate_transformation"); - // const csSchema = await getSchema(version, "coordinate_systems"); - const versionSchema = await getSchema(getSchemaUrl("version", version)); + const versionSchema = await getSchema(getSchemaUrl("_version", version)); + // const schemaSchema = await getSchema(getSchemaUrl("_schema_url", version)); refSchemas = [versionSchema]; } let errors = []; - for (let s=0; s<schemaUrls.length; s++) { - let schema = await getSchema(schemaUrls[s]); - let errs = validateData(schema, jsonData, refSchemas); - errors = errors.concat(errs); + if (jsonData.attributes) { + for (let s=0; s<schemaUrls.length; s++) { + let schema = await getSchema(schemaUrls[s]); + let errs = validateData(schema, jsonData.attributes, refSchemas); + errors = errors.concat(errs); + } + } else { + errors.push("No 'attributes' key found in JSON data"); + } + if (errors.length > 0) { + console.log("Validation errors", errors, jsonData); } return errors; } From 4341aaae6caa490f4a70497c446e190325664155 Mon Sep 17 00:00:00 2001 From: William Moore <w.moore@dundee.ac.uk> Date: Fri, 27 Sep 2024 14:14:40 +0100 Subject: [PATCH 30/33] remove console.trace() --- src/utils.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/utils.js b/src/utils.js index 99c6904..0889e13 100644 --- a/src/utils.js +++ b/src/utils.js @@ -140,7 +140,6 @@ export async function getSchema(schemaUrl) { } export function getVersion(ngffData) { - // console.log("getVersion...", ngffData, 'attributes?', ngffData.attributes) // if we have attributes.ome then this is version 0.5+ if (ngffData.attributes?.ome) { if (ngffData.attributes.ome.version) { @@ -152,11 +151,9 @@ export function getVersion(ngffData) { // Used if we have our 'attributes' at the root if (ngffData.ome?.version) { - console.trace("WARNING - using ngffData.ome?.version FIXME?") return ngffData.ome.version; } if (ngffData.version) { - console.trace("WARNING - using ngffData.version FIXME?") return ngffData.version; } // Handle version 0.4 and earlier From eb21241f0d4c7dac79ea4fdc5e07ea43674197b6 Mon Sep 17 00:00:00 2001 From: William Moore <w.moore@dundee.ac.uk> Date: Fri, 27 Sep 2024 17:22:48 +0100 Subject: [PATCH 31/33] Only validate jsonData.attributes for v0.5 data --- src/utils.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/utils.js b/src/utils.js index 0889e13..39da1c2 100644 --- a/src/utils.js +++ b/src/utils.js @@ -267,17 +267,17 @@ export async function validate(jsonData) { const versionSchema = await getSchema(getSchemaUrl("_version", version)); // const schemaSchema = await getSchema(getSchemaUrl("_schema_url", version)); refSchemas = [versionSchema]; + // For version 0.5+, we validate the "attributes" content. + // If no "attributes" exist, then it will be assumed this is v0.4 data (see above) + jsonData = jsonData.attributes; } let errors = []; - if (jsonData.attributes) { - for (let s=0; s<schemaUrls.length; s++) { - let schema = await getSchema(schemaUrls[s]); - let errs = validateData(schema, jsonData.attributes, refSchemas); - errors = errors.concat(errs); - } - } else { - errors.push("No 'attributes' key found in JSON data"); + for (let s=0; s<schemaUrls.length; s++) { + let schema = await getSchema(schemaUrls[s]); + let errs = validateData(schema, jsonData, refSchemas); + errors = errors.concat(errs); } + if (errors.length > 0) { console.log("Validation errors", errors, jsonData); } From b1216b66c262490e03e1797e463fce045e505ba5 Mon Sep 17 00:00:00 2001 From: William Moore <w.moore@dundee.ac.uk> Date: Tue, 1 Oct 2024 22:39:32 +0100 Subject: [PATCH 32/33] Handle /OME/zarr.json series metadata for bf2raw data --- src/Bioformats2rawLayout/index.svelte | 57 ++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/src/Bioformats2rawLayout/index.svelte b/src/Bioformats2rawLayout/index.svelte index 5575722..19387e8 100644 --- a/src/Bioformats2rawLayout/index.svelte +++ b/src/Bioformats2rawLayout/index.svelte @@ -1,5 +1,5 @@ <script> - import { getXmlDom, getZarrGroupAttrs, validate, getVersion, getZarrGroupAttrsFileName } from "../utils"; + import { getJson, getXmlDom, getZarrGroupAttrs, validate, getVersion, getZarrGroupAttrsFileName } from "../utils"; import JsonBrowser from "../JsonBrowser/index.svelte"; import ImageContainer from "../JsonValidator/Well/ImageContainer.svelte"; import RoCrate from "../JsonValidator/RoCrate/index.svelte"; @@ -17,17 +17,56 @@ // source/OME/METADATA.ome.xml const metadataUrl = `${source}/${metadataName}`; - async function loadXml(url) { - let dom = await getXmlDom(url); + async function loadXmlOrSeries(url) { + // We can get the series info from /OME/METADATA.ome.xml or + // /OME/zarr.json group attributes + // returns {images: [{name, id}]} or {error: message} + let dom; + try { + dom = await getXmlDom(url); + } catch (error) {} + let xmlRsp; + if (dom) { + xmlRsp = parseXml(dom); + } + // Try to get series info + let rsp = {}; + let series; + try { + let zarrAttrs = await getJson(`${source}/OME/${zarrAttrsFileName}`); + series = zarrAttrs?.attributes?.ome?.series || zarrAttrs.series; + } catch (error) {} + + if (series && xmlRsp && xmlRsp?.images) { + // MUST match if we have both... + if (series.length !== xmlRsp.images.length) { + rsp.errors = [ + `Length mismatch: series: ${series.length} != ome.xml: ${5}`, + ]; + } + } + if (series) { + rsp.images = series.map((seriesName) => ({ name: seriesName, path: seriesName })); + } else if (xmlRsp) { + rsp = xmlRsp; + } else { + rsp.errors = ["No OME/METADATA.ome.xml or /OME series found"]; + } + return rsp; + } + + function parseXml(dom) { const root = dom.documentElement; let rsp = { images: [] }; + let index = 0; for (const child of root.children) { console.log(child.tagName); if (child.tagName === "Image") { rsp.images.push({ name: child.getAttribute("Name"), id: child.getAttribute("ID"), + path: "" + index++, }); } // error handling - parsererror gives html doc @@ -41,7 +80,7 @@ } return rsp; } - const promise = loadXml(metadataUrl); + const promise = loadXmlOrSeries(metadataUrl); // wait for schema to be cached, so we don't load them multiple times // let schemasPromise = getSchema("0.2", "image"); @@ -77,14 +116,14 @@ <div>loading schema...</div> {:then ok} <ul> - {#each metadataJson.images as image, i} + {#each metadataJson.images as image} <li class="image"> - /{i} - <a title="Open Image" href="{url}?source={source}/{i}/" + /{image.path} + <a title="Open Image" href="{url}?source={source}/{image.path}/" >{image.name}</a > - <ImageContainer {source} path={i} /> + <ImageContainer {source} path={image.path} /> </li> {/each} </ul> @@ -96,7 +135,7 @@ <!-- Error handling... --> {#if metadataJson.errors} <div class="error"> - <h2>Error parsing {metadataName}</h2> + <h3>Error loading series metadata:</h3> {#each metadataJson.errors as err, i} <div>{err}</div> {/each} From 2853654ce9c93803fe9e1ecae3f7620f23dc6f60 Mon Sep 17 00:00:00 2001 From: William Moore <w.moore@dundee.ac.uk> Date: Wed, 2 Oct 2024 07:22:20 +0100 Subject: [PATCH 33/33] Fix bytesPerPixel for Zarr v3 dtypes --- .../MultiscaleArrays/ZarrArray/index.svelte | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte b/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte index e32add8..88109b0 100644 --- a/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte +++ b/src/JsonValidator/MultiscaleArrays/ZarrArray/index.svelte @@ -32,8 +32,16 @@ function getBytes(shape, zarray) { // handle v2 and v3 zarr - let dtype = getArrayDtype(zarray) - let bytesPerPixel = [1, 2, 4, 8].find((n) => dtype.includes(n)); + let dtype = getArrayDtype(zarray); + let bytesPerPixel; + // e.g. v3: uint8, uint16, uint32, uint64, int8, int16, int32 (4), int64 (8), float32 (4), float64 (8) + if (dtype.includes("int") || dtype.includes("float")) { + // use regex to get numbers from dtype + bytesPerPixel = parseInt(dtype.match(/\d+/)[0]) / 8; + } else { + // e.g. v2: >i1, >i2, >i4, >i8, >u1, >u2, >u4, >u8, >f4, >f8 + bytesPerPixel = [1, 2, 4, 8].find((n) => dtype.includes(n)); + } if (!bytesPerPixel) return ""; let pixels = shape.reduce((i, p) => i * p, 1); return formatBytes(bytesPerPixel * pixels);