From 5cd19e39909570bb1b1dbc9c891122cdc711290b Mon Sep 17 00:00:00 2001 From: Johannes Liem Date: Tue, 29 Aug 2023 13:56:47 +0200 Subject: [PATCH] adding the option to add upstream ids to event tabels in xlsx template (as relatedEntity with some role) --- .changeset/nine-otters-wait.md | 5 ++ package-lock.json | 85 +++++++++++++++++++++++++++++++++- package.json | 1 + src/config.ts | 1 - src/create-event.ts | 42 +++++++++++++---- src/import-data.ts | 1 + src/transform-data.ts | 21 ++++++++- 7 files changed, 143 insertions(+), 13 deletions(-) create mode 100644 .changeset/nine-otters-wait.md diff --git a/.changeset/nine-otters-wait.md b/.changeset/nine-otters-wait.md new file mode 100644 index 0000000..d8ae699 --- /dev/null +++ b/.changeset/nine-otters-wait.md @@ -0,0 +1,5 @@ +--- +"@intavia/data-import": patch +--- + +adding the option to add upstream ids to event tabels in xlsx template (as relatedEntity with some role) diff --git a/package-lock.json b/package-lock.json index 77bf5f8..8c047be 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "@intavia/data-import", - "version": "0.1.9", + "version": "0.1.10", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@intavia/data-import", - "version": "0.1.9", + "version": "0.1.10", "license": "MIT", "dependencies": { "@intavia/api-client": "^0.1.25", + "buffer": "^6.0.3", "lodash.isequal": "^4.5.0", "xlsx": "^0.18.5" }, @@ -3153,6 +3154,25 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/better-path-resolve": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/better-path-resolve/-/better-path-resolve-1.0.0.tgz", @@ -3242,6 +3262,29 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -5217,6 +5260,25 @@ "postcss": "^8.1.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", @@ -11562,6 +11624,11 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "better-path-resolve": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/better-path-resolve/-/better-path-resolve-1.0.0.tgz", @@ -11626,6 +11693,15 @@ "update-browserslist-db": "^1.0.9" } }, + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -12994,6 +13070,11 @@ "dev": true, "requires": {} }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "ignore": { "version": "5.2.4", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz", diff --git a/package.json b/package.json index ca0cc5f..b3f056a 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ }, "dependencies": { "@intavia/api-client": "^0.1.25", + "buffer": "^6.0.3", "lodash.isequal": "^4.5.0", "xlsx": "^0.18.5" }, diff --git a/src/config.ts b/src/config.ts index 1c3b0c3..3f9ce60 100644 --- a/src/config.ts +++ b/src/config.ts @@ -32,7 +32,6 @@ export const eventTargetProps: Array = [ "source", "startDate", "endDate", - "relations", ]; export const biographyTargetProps: Array = ["id", "text", "citation"]; diff --git a/src/create-event.ts b/src/create-event.ts index 0dd80c4..4fcf486 100644 --- a/src/create-event.ts +++ b/src/create-event.ts @@ -9,10 +9,12 @@ import type { import { eventTargetProps } from "./config"; import { eventPropertyMappers } from "./data-mappers"; import type { UnmappedProps, VocabularyIdAndEntry } from "./types"; +import { Buffer } from "buffer"; interface CreateEventReturn { event: Event; vocabularyEntries?: Array; + upstreamEntities?: Array; unmappedProperties?: UnmappedProps; } @@ -28,6 +30,7 @@ export function createEvent(entry: Record): CreateEventReturn { const event: any = {}; const vocabularyEntries: Array = []; + const upstreamEntities: Array = []; for (const targetProp of targetProps) { if (!(targetProp in eventPropertyMappers)) { @@ -65,6 +68,19 @@ export function createEvent(entry: Record): CreateEventReturn { } } + // eventEntityRelation + const { + eventEntityRelation, + vocabularyEntries: _vocabularyEntries, + upstreamEntities: _upstreamEntities, + } = createEventEntityRelation(entry); + if (!("relations" in event)) { + event.relations = []; + } + eventEntityRelation !== undefined && event.relations.push(eventEntityRelation); + _vocabularyEntries !== undefined && vocabularyEntries.push(..._vocabularyEntries); + _upstreamEntities !== undefined && upstreamEntities.push(..._upstreamEntities); + // has Place ? add additional EventEntityRelation if (entry.place != null && String(entry.place).trim().length > 0) { for (const place of String(entry.place).split(";")) { @@ -97,6 +113,7 @@ export function createEvent(entry: Record): CreateEventReturn { return { event, vocabularyEntries, + upstreamEntities, unmappedProperties, }; } @@ -104,6 +121,7 @@ export function createEvent(entry: Record): CreateEventReturn { interface CreateEventEntityRelationReturn { eventEntityRelation?: EventEntityRelation; vocabularyEntries?: Array; + upstreamEntities?: Array; error?: string; } @@ -121,14 +139,22 @@ export function createEventEntityRelation( return { error: "some error" }; } - if (mapper.vocabulary !== undefined) { - return { - eventEntityRelation: mapper.mapper(entry)[0], - vocabularyEntries: [mapper.vocabulary(entry)], - }; + const eventEntityRelation = mapper.mapper(entry)[0]; + const base64candidate = eventEntityRelation.entity.split("-")[1]; + const base64regex = /^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$/; + const isUpstream = + base64regex.test(base64candidate) && + Buffer.from(base64candidate, "base64").toString("utf-8").startsWith("http"); + + const result: CreateEventEntityRelationReturn = {}; + + if (isUpstream) { + eventEntityRelation.entity = base64candidate; + result.upstreamEntities = [eventEntityRelation.entity]; } - return { - eventEntityRelation: mapper.mapper(entry)[0], - }; + eventEntityRelation && (result.eventEntityRelation = eventEntityRelation); + mapper.vocabulary !== undefined && (result.vocabularyEntries = [mapper.vocabulary(entry)]); + + return result; } diff --git a/src/import-data.ts b/src/import-data.ts index 8ba5ca1..203169e 100644 --- a/src/import-data.ts +++ b/src/import-data.ts @@ -19,6 +19,7 @@ export interface ImportData { unmappedEntries?: Array; collections?: Record; tags?: Array; + upstreamEntityIds?: Array; } interface ImportDataParams { diff --git a/src/transform-data.ts b/src/transform-data.ts index e51abba..7bc2fee 100644 --- a/src/transform-data.ts +++ b/src/transform-data.ts @@ -27,6 +27,7 @@ export function transformData(params: TransformDataParams): ImportData { const entityGroups: Record> = {}; const collections: ImportData["collections"] = {}; const tags: ImportData["tags"] = []; + const upstreamEntityIds: ImportData["upstreamEntityIds"] = []; const registerVocabularyEntries = ( vocabularyEntries: Array | undefined @@ -44,6 +45,18 @@ export function transformData(params: TransformDataParams): ImportData { } }; + const registerUpstreamEntities = ( + _upstreamEntities: ImportData["upstreamEntityIds"] | undefined + ) => { + if (_upstreamEntities !== undefined) { + for (const upstreamEntity of _upstreamEntities) { + if (!upstreamEntityIds.includes(upstreamEntity)) { + upstreamEntityIds.push(upstreamEntity); + } + } + } + }; + // check if any of the entries has tagging // has Tag for all? otherwise use prefix for all entities and events let tagsForAll = input.filter((entry) => { @@ -129,12 +142,13 @@ export function transformData(params: TransformDataParams): ImportData { // Add relation to existing event (rows in the xlsx that have the same id an only add relations; other props are ignored) events = events.map((event) => { if (event.id === entry.id) { - const { eventEntityRelation, vocabularyEntries } = + const { eventEntityRelation, vocabularyEntries, upstreamEntities } = createEventEntityRelation(entry); eventEntityRelation !== undefined && event.relations.push(eventEntityRelation); registerVocabularyEntries(vocabularyEntries); + registerUpstreamEntities(upstreamEntities); // TODO unmapped entries } @@ -142,9 +156,11 @@ export function transformData(params: TransformDataParams): ImportData { }); } else { // Create new event - const { event, vocabularyEntries, unmappedProperties } = createEvent(entry); + const { event, vocabularyEntries, upstreamEntities, unmappedProperties } = + createEvent(entry); events.push(event); registerVocabularyEntries(vocabularyEntries); + registerUpstreamEntities(upstreamEntities); const entryGroup = entry.sheetName as string; if (!(entryGroup in eventGroups)) { @@ -272,6 +288,7 @@ export function transformData(params: TransformDataParams): ImportData { unmappedEntries && (result.unmappedEntries = unmappedEntries); collections && (result.collections = collections); tags && (result.tags = _tags); + upstreamEntityIds && (result.upstreamEntityIds = upstreamEntityIds); return result; }