Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

GH-5080 Support for using a user provided document loader with HASMAC JSON-LD #5162

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

GH-5080 Support for using a user provided document loader with HASMAC JSON-LD #5162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d66ac9b
GH-5080 duplicate all settings so we can change the document loader s…
hmottestad Oct 28, 2024
2582602
code cleanup
hmottestad Oct 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions core/rio/api/src/main/java/org/eclipse/rdf4j/rio/RioConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@
package org.eclipse.rdf4j.rio;

import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
Expand Down Expand Up @@ -152,4 +156,8 @@ public RioConfig useDefaults() {
systemPropertyCache.clear();
return this;
}

public Map<RioSetting<Object>, Object> getSettings() {
return Collections.unmodifiableMap(new HashMap<>(settings));
}
}
75 changes: 58 additions & 17 deletions core/rio/api/src/main/java/org/eclipse/rdf4j/rio/helpers/JSONLDSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,13 @@ public class JSONLDSettings {
* Can be overridden by setting system property {@code org.eclipse.rdf4j.rio.jsonld.compact_arrays}.
*
* @see <a href="http://json-ld.org/spec/latest/json-ld-api/#data-structures">JSONLD Data Structures</a>
*
*/
public static final BooleanRioSetting COMPACT_ARRAYS = new BooleanRioSetting(
"org.eclipse.rdf4j.rio.jsonld.compact_arrays", "Compact arrays", Boolean.TRUE);

/**
* If specified, it is used to retrieve remote documents and contexts; otherwise the processor's built-in loader is
* used.
*
*/
public static final RioSetting<DocumentLoader> DOCUMENT_LOADER = new ClassRioSetting<>(
"org.eclipse.rdf4j.rio.jsonld.document_loader", "Document loader", null);
Expand All @@ -66,7 +64,6 @@ public class JSONLDSettings {

/**
* The JSON-LD processor will throw an exception if a warning is encountered during processing.
*
*/
public static final BooleanRioSetting EXCEPTION_ON_WARNING = new BooleanRioSetting(
"org.eclipse.rdf4j.rio.jsonld.exception_on_warning",
Expand All @@ -83,7 +80,6 @@ public class JSONLDSettings {
* Can be overridden by setting system property {@code org.eclipse.rdf4j.rio.jsonld.optimize}.
*
* @see <a href="http://json-ld.org/spec/latest/json-ld-api/#data-structures">JSONLD Data Structures</a>
*
*/
public static final BooleanRioSetting OPTIMIZE = new BooleanRioSetting("org.eclipse.rdf4j.rio.jsonld.optimize",
"Optimize output", Boolean.FALSE);
Expand All @@ -99,7 +95,6 @@ public class JSONLDSettings {
* Can be overridden by setting system property {@code org.eclipse.rdf4j.rio.jsonld.produce_generalized_rdf}.
*
* @see <a href="http://json-ld.org/spec/latest/json-ld-api/#data-structures">JSONLD Data Structures</a>
*
*/
public static final BooleanRioSetting PRODUCE_GENERALIZED_RDF = new BooleanRioSetting(
"org.eclipse.rdf4j.rio.jsonld.produce_generalized_rdf", "Produce generalized RDF", Boolean.FALSE);
Expand All @@ -114,7 +109,6 @@ public class JSONLDSettings {
* Can be overridden by setting system property {@code org.eclipse.rdf4j.rio.jsonld.use_native_types}.
*
* @see <a href="http://json-ld.org/spec/latest/json-ld-api/#data-structures">JSONLD Data Structures</a>
*
*/
public static final BooleanRioSetting USE_NATIVE_TYPES = new BooleanRioSetting(
"org.eclipse.rdf4j.rio.jsonld.use_native_types", "Use Native JSON Types", Boolean.FALSE);
Expand All @@ -128,7 +122,6 @@ public class JSONLDSettings {
* Can be overridden by setting system property {@code org.eclipse.rdf4j.rio.jsonld.use_rdf_type}.
*
* @see <a href="http://json-ld.org/spec/latest/json-ld-api/#data-structures">JSONLD Data Structures</a>
*
*/
public static final BooleanRioSetting USE_RDF_TYPE = new BooleanRioSetting(
"org.eclipse.rdf4j.rio.jsonld.use_rdf_type", "Use RDF Type", Boolean.FALSE);
Expand All @@ -139,7 +132,6 @@ public class JSONLDSettings {
* Defaults to {@link JSONLDMode#EXPAND} to provide maximum RDF compatibility.
*
* @see <a href="http://json-ld.org/spec/latest/json-ld-api/#features">JSONLD Features</a>
*
*/
public static final RioSetting<JSONLDMode> JSONLD_MODE = new RioSettingImpl<>(
"org.eclipse.rdf4j.rio.jsonld_mode", "JSONLD Mode", JSONLDMode.EXPAND);
Expand All @@ -150,7 +142,6 @@ public class JSONLDSettings {
* Default to false
* <p>
* Can be overridden by setting system property {@code org.eclipse.rdf4j.rio.jsonld.hierarchical_view}.
*
*/
public static final BooleanRioSetting HIERARCHICAL_VIEW = new BooleanRioSetting(
"org.eclipse.rdf4j.rio.jsonld.hierarchical_view", "Hierarchical representation of the JSON", Boolean.FALSE);
Expand All @@ -161,38 +152,88 @@ public class JSONLDSettings {
* array of the desired values.
* <p>
* Default:
* {@code Set.of("http://www.w3.org/ns/anno.jsonld", "http://www.w3.org/ns/activitystreams.jsonld", "http://www.w3.org/ns/ldp.jsonld", "http://www.w3.org/ns/oa.jsonld", "http://www.w3.org/ns/hydra/context.jsonld", "http://schema.org/", "https://w3id.org/security/v1", "https://w3c.github.io/json-ld-rc/context.jsonld", "https://www.w3.org/2018/credentials/v1", "https://health-lifesci.schema.org/", "https://auto.schema.org/", "https://bib.schema.org/", "http://xmlns.com/foaf/spec/index.jsonld", "https://pending.schema.org/", "https://schema.org/", "https://schema.org/docs/jsonldcontext.jsonld", "https://schema.org/version/latest/schemaorg-current-https.jsonld", "https://schema.org/version/latest/schemaorg-all-http.jsonld", "https://schema.org/version/latest/schemaorg-all-https.jsonld", "https://schema.org/version/latest/schemaorg-current-http.jsonld", "https://schema.org/version/latest/schemaorg-all.jsonld", "https://schema.org/version/latest/schemaorg-current.jsonld", "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld", "https://geojson.org/geojson-ld/geojson-context.jsonld", "https://www.w3.org/2019/wot/td/v1");
*
* {@code Set.of("http://www.w3.org/ns/anno.jsonld", "https://www.w3.org/ns/anno.jsonld", "http://www.w3.org/ns/anno", "https://www.w3.org/ns/anno", "http://www.w3.org/ns/activitystreams.jsonld", "https://www.w3.org/ns/activitystreams.jsonld", "http://www.w3.org/ns/activitystreams", "https://www.w3.org/ns/activitystreams", "http://www.w3.org/ns/ldp.jsonld", "https://www.w3.org/ns/ldp.jsonld", "http://www.w3.org/ns/ldp", "https://www.w3.org/ns/ldp", "http://www.w3.org/ns/oa.jsonld", "https://www.w3.org/ns/oa.jsonld", "http://www.w3.org/ns/oa", "https://www.w3.org/ns/oa", "http://www.w3.org/ns/hydra/context.jsonld", "https://www.w3.org/ns/hydra/context.jsonld", "http://www.w3.org/ns/hydra/context", "https://www.w3.org/ns/hydra/context", "http://www.w3.org/2018/credentials/v1.jsonld", "https://www.w3.org/2018/credentials/v1.jsonld", "http://www.w3.org/2018/credentials/v1", "https://www.w3.org/2018/credentials/v1", "http://www.w3.org/2019/wot/td/v1.jsonld", "https://www.w3.org/2019/wot/td/v1.jsonld", "http://www.w3.org/2019/wot/td/v1", "https://www.w3.org/2019/wot/td/v1", "http://w3c.github.io/json-ld-rc/context.jsonld", "https://w3c.github.io/json-ld-rc/context.jsonld", "http://schema.org/", "https://schema.org/", "http://health-lifesci.schema.org/", "https://health-lifesci.schema.org/", "http://auto.schema.org/", "https://auto.schema.org/", "http://bib.schema.org/", "https://bib.schema.org/", "http://pending.schema.org/", "https://pending.schema.org/", "http://schema.org", "https://schema.org", "http://health-lifesci.schema.org", "https://health-lifesci.schema.org", "http://auto.schema.org", "https://auto.schema.org", "http://bib.schema.org", "https://bib.schema.org", "http://pending.schema.org", "https://pending.schema.org", "http://schema.org/docs/jsonldcontext.jsonld", "https://schema.org/docs/jsonldcontext.jsonld", "http://schema.org/version/latest/schemaorg-current-https.jsonld", "https://schema.org/version/latest/schemaorg-current-https.jsonld", "http://schema.org/version/latest/schemaorg-all-http.jsonld", "https://schema.org/version/latest/schemaorg-all-http.jsonld", "http://schema.org/version/latest/schemaorg-all-https.jsonld", "https://schema.org/version/latest/schemaorg-all-https.jsonld", "http://schema.org/version/latest/schemaorg-current-http.jsonld", "https://schema.org/version/latest/schemaorg-current-http.jsonld", "http://schema.org/version/latest/schemaorg-all.jsonld", "https://schema.org/version/latest/schemaorg-all.jsonld", "http://schema.org/version/latest/schemaorg-current.jsonld", "https://schema.org/version/latest/schemaorg-current.jsonld", "http://project-open-data.cio.gov/v1.1/schema/catalog.jsonld", "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld", "http://geojson.org/geojson-ld/geojson-context.jsonld", "https://geojson.org/geojson-ld/geojson-context.jsonld", "http://w3id.org/security/v1", "https://w3id.org/security/v1", "http://xmlns.com/foaf/spec/index.jsonld", "https://xmlns.com/foaf/spec/index.jsonld", "http://xmlns.com/foaf/spec/", "https://xmlns.com/foaf/spec/", "http://xmlns.com/foaf/spec", "https://xmlns.com/foaf/spec")}
*/
public static final SetRioSetting<String> WHITELIST = new SetRioSetting<>(
"org.eclipse.rdf4j.rio.jsonld_whitelist",
"Whitelist of remote/local resources that the JSON-LD parser can retrieve. Set of URIs as strings.",
Set.of(
"http://www.w3.org/ns/anno.jsonld",
"https://www.w3.org/ns/anno.jsonld",
"http://www.w3.org/ns/anno",
"https://www.w3.org/ns/anno",
"http://www.w3.org/ns/activitystreams.jsonld",
"https://www.w3.org/ns/activitystreams.jsonld",
"http://www.w3.org/ns/activitystreams",
"https://www.w3.org/ns/activitystreams",
"http://www.w3.org/ns/ldp.jsonld",
"https://www.w3.org/ns/ldp.jsonld",
"http://www.w3.org/ns/ldp",
"https://www.w3.org/ns/ldp",
"http://www.w3.org/ns/oa.jsonld",
"https://www.w3.org/ns/oa.jsonld",
"http://www.w3.org/ns/oa",
"https://www.w3.org/ns/oa",
"http://www.w3.org/ns/hydra/context.jsonld",
"http://schema.org/",
"https://w3id.org/security/v1",
"https://w3c.github.io/json-ld-rc/context.jsonld",
"https://www.w3.org/ns/hydra/context.jsonld",
"http://www.w3.org/ns/hydra/context",
"https://www.w3.org/ns/hydra/context",
"http://www.w3.org/2018/credentials/v1.jsonld",
"https://www.w3.org/2018/credentials/v1.jsonld",
"http://www.w3.org/2018/credentials/v1",
"https://www.w3.org/2018/credentials/v1",
"http://www.w3.org/2019/wot/td/v1.jsonld",
"https://www.w3.org/2019/wot/td/v1.jsonld",
"http://www.w3.org/2019/wot/td/v1",
"https://www.w3.org/2019/wot/td/v1",
"http://w3c.github.io/json-ld-rc/context.jsonld",
"https://w3c.github.io/json-ld-rc/context.jsonld",
"http://schema.org/",
"https://schema.org/",
"http://health-lifesci.schema.org/",
"https://health-lifesci.schema.org/",
"http://auto.schema.org/",
"https://auto.schema.org/",
"http://bib.schema.org/",
"https://bib.schema.org/",
"http://xmlns.com/foaf/spec/index.jsonld",
"http://pending.schema.org/",
"https://pending.schema.org/",
"https://schema.org/",
"http://schema.org",
"https://schema.org",
"http://health-lifesci.schema.org",
"https://health-lifesci.schema.org",
"http://auto.schema.org",
"https://auto.schema.org",
"http://bib.schema.org",
"https://bib.schema.org",
"http://pending.schema.org",
"https://pending.schema.org",
"http://schema.org/docs/jsonldcontext.jsonld",
"https://schema.org/docs/jsonldcontext.jsonld",
"http://schema.org/version/latest/schemaorg-current-https.jsonld",
"https://schema.org/version/latest/schemaorg-current-https.jsonld",
"http://schema.org/version/latest/schemaorg-all-http.jsonld",
"https://schema.org/version/latest/schemaorg-all-http.jsonld",
"http://schema.org/version/latest/schemaorg-all-https.jsonld",
"https://schema.org/version/latest/schemaorg-all-https.jsonld",
"http://schema.org/version/latest/schemaorg-current-http.jsonld",
"https://schema.org/version/latest/schemaorg-current-http.jsonld",
"http://schema.org/version/latest/schemaorg-all.jsonld",
"https://schema.org/version/latest/schemaorg-all.jsonld",
"http://schema.org/version/latest/schemaorg-current.jsonld",
"https://schema.org/version/latest/schemaorg-current.jsonld",
"http://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
"http://geojson.org/geojson-ld/geojson-context.jsonld",
"https://geojson.org/geojson-ld/geojson-context.jsonld",
"https://www.w3.org/2019/wot/td/v1"
"http://w3id.org/security/v1",
"https://w3id.org/security/v1",
"http://xmlns.com/foaf/spec/index.jsonld",
"https://xmlns.com/foaf/spec/index.jsonld",
"http://xmlns.com/foaf/spec/",
"https://xmlns.com/foaf/spec/",
"http://xmlns.com/foaf/spec",
"https://xmlns.com/foaf/spec"
));

/**
Expand Down
2 changes: 1 addition & 1 deletion core/rio/jsonld/src/main/java/org/eclipse/rdf4j/rio/jsonld/CachingDocumentLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public class CachingDocumentLoader implements DocumentLoader {
private static final LoadingCache<URI, Document> cache = CacheBuilder.newBuilder()
.maximumSize(1000) // Maximum 1000 documents in cache
.expireAfterWrite(1, TimeUnit.HOURS) // Expire after 1 hour
.concurrencyLevel(8) // Optimize for 8 concurrent threads
.concurrencyLevel(Runtime.getRuntime().availableProcessors())
.build(new CacheLoader<>() {
@Override
public Document load(URI url) throws Exception {
Expand Down
54 changes: 54 additions & 0 deletions core/rio/jsonld/src/main/java/org/eclipse/rdf4j/rio/jsonld/JSONLDMode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*******************************************************************************
* Copyright (c) 2015 Eclipse RDF4J contributors, Aduna, and others.
*
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Distribution License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*******************************************************************************/
package org.eclipse.rdf4j.rio.jsonld;

/**
* Specifies constants to identify various modes that are relevant to JSONLD documents.
*
* @author Peter Ansell
* @see <a href="http://json-ld.org/spec/latest/json-ld-api/#features">JSONLD Features</a>
*
*/
public enum JSONLDMode {

EXPAND("Expansion", "http://json-ld.org/spec/latest/json-ld-api/index.html#expansion"),

COMPACT("Compaction", "http://json-ld.org/spec/latest/json-ld-api/index.html#compaction"),

FLATTEN("Flattening", "http://json-ld.org/spec/latest/json-ld-api/index.html#flattening"),

FRAME("Framing", "https://www.w3.org/TR/json-ld11-framing/"),

;

private final String label;

private final String reference;

JSONLDMode(String label, String reference) {
this.label = label;
this.reference = reference;
}

/**
* @return Returns the label.
*/
public String getLabel() {
return label;
}

/**
* @return Returns the reference URL for the given mode.
*/
public String getReference() {
return reference;
}
}
36 changes: 23 additions & 13 deletions core/rio/jsonld/src/main/java/org/eclipse/rdf4j/rio/jsonld/JSONLDParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,13 @@
*******************************************************************************/
package org.eclipse.rdf4j.rio.jsonld;

import static org.eclipse.rdf4j.rio.helpers.JSONLDSettings.DOCUMENT_LOADER_CACHE;
import static org.eclipse.rdf4j.rio.helpers.JSONLDSettings.SECURE_MODE;
import static org.eclipse.rdf4j.rio.helpers.JSONLDSettings.WHITELIST;

import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Collection;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.BiConsumer;
Expand All @@ -35,10 +32,11 @@
import org.eclipse.rdf4j.rio.RDFHandlerException;
import org.eclipse.rdf4j.rio.RDFParseException;
import org.eclipse.rdf4j.rio.RDFParser;
import org.eclipse.rdf4j.rio.RioConfig;
import org.eclipse.rdf4j.rio.RioSetting;
import org.eclipse.rdf4j.rio.helpers.AbstractRDFParser;
import org.eclipse.rdf4j.rio.helpers.BasicParserSettings;
import org.eclipse.rdf4j.rio.helpers.JSONLDSettings;
import org.eclipse.rdf4j.rio.helpers.BasicWriterSettings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -91,6 +89,13 @@ public Collection<RioSetting<?>> getSupportedSettings() {

result.add(JSONLDSettings.EXPAND_CONTEXT);
result.add(JSONLDSettings.EXCEPTION_ON_WARNING);
result.add(JSONLDSettings.SECURE_MODE);
result.add(JSONLDSettings.WHITELIST);
result.add(JSONLDSettings.DOCUMENT_LOADER);
result.add(JSONLDSettings.DOCUMENT_LOADER_CACHE);
result.add(org.eclipse.rdf4j.rio.helpers.JSONLDSettings.SECURE_MODE);
result.add(org.eclipse.rdf4j.rio.helpers.JSONLDSettings.WHITELIST);
result.add(org.eclipse.rdf4j.rio.helpers.JSONLDSettings.DOCUMENT_LOADER_CACHE);

return result;
}
Expand Down Expand Up @@ -133,9 +138,9 @@ private void parse(InputStream in, Reader reader, String baseURI)
BasicParserSettings.FAIL_ON_UNKNOWN_LANGUAGES);
}

boolean secureMode = getParserConfig().get(SECURE_MODE);
Set<String> whitelist = getParserConfig().get(WHITELIST);
boolean documentLoaderCache = getParserConfig().get(DOCUMENT_LOADER_CACHE);
boolean secureMode = getParserConfig().get(JSONLDSettings.SECURE_MODE);
Set<String> whitelist = getParserConfig().get(JSONLDSettings.WHITELIST);
boolean documentLoaderCache = getParserConfig().get(JSONLDSettings.DOCUMENT_LOADER_CACHE);

JsonLdOptions opts = new JsonLdOptions();
opts.setUriValidation(false);
Expand All @@ -144,8 +149,13 @@ private void parse(InputStream in, Reader reader, String baseURI)
Document context = getParserConfig().get(JSONLDSettings.EXPAND_CONTEXT);

DocumentLoader defaultDocumentLoader = opts.getDocumentLoader();
CachingDocumentLoader cachingDocumentLoader = new CachingDocumentLoader(secureMode, whitelist,
documentLoaderCache);

DocumentLoader documentLoader;
if (getParserConfig().get(JSONLDSettings.DOCUMENT_LOADER) == null) {
documentLoader = new CachingDocumentLoader(secureMode, whitelist, documentLoaderCache);
} else {
documentLoader = getParserConfig().get(JSONLDSettings.DOCUMENT_LOADER);
}

if (context != null) {

Expand All @@ -162,14 +172,14 @@ private void parse(InputStream in, Reader reader, String baseURI)
return context;
}

return cachingDocumentLoader.loadDocument(uri, options);
return documentLoader.loadDocument(uri, options);
});
}

}

if (secureMode && opts.getDocumentLoader() == defaultDocumentLoader) {
opts.setDocumentLoader(cachingDocumentLoader);
if (opts.getDocumentLoader() == defaultDocumentLoader) {
opts.setDocumentLoader(documentLoader);
}

if (baseURI != null && !baseURI.isEmpty()) {
Expand Down
Loading
Loading