diff --git a/schema_salad/metaschema/metaschema.yml b/schema_salad/metaschema/metaschema.yml index d4ec75b51..3de1f9dcc 100644 --- a/schema_salad/metaschema/metaschema.yml +++ b/schema_salad/metaschema/metaschema.yml @@ -194,10 +194,8 @@ $graph: fields: - name: doc type: - - "null" - - string - - type: array - items: string + - string? + - string[]? doc: "A documentation string for this type, or an array of strings which should be concatenated." jsonldPredicate: "sld:doc" @@ -212,10 +210,8 @@ $graph: - name: docChild type: - - "null" - - string - - type: array - items: string + - string? + - string[]? doc: | Hint to indicate that during documentation generation, documentation for `docChild` should appear in a subsection under this type. @@ -236,19 +232,18 @@ $graph: - name: SchemaDefinedType type: record - extends: "#DocType" + extends: DocType doc: | Abstract base for schema-defined types. abstract: true fields: - name: jsonldPredicate type: - - "null" - - string - - "#JsonldPredicate" + - string? + - JsonldPredicate? doc: | Annotate this type with linked data context. - jsonldPredicate: "sld:jsonldPredicate" + jsonldPredicate: sld:jsonldPredicate - name: documentRoot type: boolean? @@ -259,7 +254,7 @@ $graph: - name: RecordField type: record - doc: "A field of a record." + doc: A field of a record. fields: - name: name type: string @@ -275,36 +270,36 @@ $graph: - name: type type: - - "#PrimitiveType" - - "#RecordSchema" - - "#EnumSchema" - - "#ArraySchema" + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema - string - type: array items: - - "#PrimitiveType" - - "#RecordSchema" - - "#EnumSchema" - - "#ArraySchema" + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema - string jsonldPredicate: - _id: "sld:type" + _id: sld:type _type: "@vocab" typeDSL: true + refScope: 0 doc: | The field type - name: SaladRecordField type: record - extends: "#RecordField" + extends: RecordField doc: "A field of a record." fields: - name: jsonldPredicate type: - - "null" - - string - - "#JsonldPredicate" + - string? + - JsonldPredicate? doc: | Annotate this type with linked data context. jsonldPredicate: "sld:jsonldPredicate" @@ -323,19 +318,16 @@ $graph: _id: "sld:type" _type: "@vocab" typeDSL: true + refScope: 0 - name: "fields" - type: - - "null" - - type: "array" - items: "#RecordField" - + type: RecordField[]? jsonldPredicate: "sld:fields" doc: "Defines the fields of the record." - name: SaladRecordSchema type: record - extends: ["#NamedType", "#RecordSchema", "#SchemaDefinedType"] + extends: [NamedType, RecordSchema, SchemaDefinedType] documentRoot: true specialize: specializeFrom: "#RecordField" @@ -349,22 +341,19 @@ $graph: - name: extends type: - - "null" - - string - - type: array - items: string + - string? + - string[]? jsonldPredicate: _id: "sld:extends" _type: "@id" + refScope: 0 doc: | Indicates that this record inherits fields from one or more base records. - name: specialize type: - - "null" - - "#SpecializeDef" - - type: array - items: "#SpecializeDef" + - SpecializeDef? + - SpecializeDef[]? doc: | Only applies if `extends` is declared. Apply type specialization using the base record as a template. For each field inherited from the base @@ -388,10 +377,9 @@ $graph: _id: "sld:type" _type: "@vocab" typeDSL: true + refScope: 0 - name: "symbols" - type: - - type: "array" - items: "string" + type: string[] jsonldPredicate: _id: "sld:symbols" _type: "@id" @@ -401,20 +389,19 @@ $graph: - name: SaladEnumSchema type: record - extends: ["#EnumSchema", "#SchemaDefinedType"] + extends: [EnumSchema, SchemaDefinedType] documentRoot: true doc: | Define an enumerated type. fields: - name: extends type: - - "null" - - string - - type: array - items: string + - string? + - string[]? jsonldPredicate: _id: "sld:extends" _type: "@id" + refScope: 0 doc: | Indicates that this enum inherits symbols from a base enum. @@ -433,29 +420,31 @@ $graph: _id: "sld:type" _type: "@vocab" typeDSL: true + refScope: 0 - name: items type: - - "#PrimitiveType" - - "#RecordSchema" - - "#EnumSchema" - - "#ArraySchema" + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema - string - type: array items: - - "#PrimitiveType" - - "#RecordSchema" - - "#EnumSchema" - - "#ArraySchema" + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema - string jsonldPredicate: _id: "sld:items" _type: "@vocab" + refScope: 0 doc: "Defines the type of the array elements." - name: Documentation type: record - extends: ["#NamedType", "#DocType"] + extends: [NamedType, DocType] documentRoot: true doc: | A documentation section. This type exists to facilitate self-documenting @@ -471,4 +460,5 @@ $graph: jsonldPredicate: _id: "sld:type" _type: "@vocab" - typeDSL: true \ No newline at end of file + typeDSL: true + refScope: 0 diff --git a/schema_salad/ref_resolver.py b/schema_salad/ref_resolver.py index 0a303ae19..4fac058e0 100644 --- a/schema_salad/ref_resolver.py +++ b/schema_salad/ref_resolver.py @@ -110,7 +110,7 @@ def __init__(self, ctx, schemagraph=None, foreign_properties=None, self.add_context(ctx) - def expand_url(self, url, base_url, scoped=False, vocab_term=False): + def expand_url(self, url, base_url, scoped_id=False, vocab_term=False, scoped_ref=None): # type: (Union[str, unicode], Union[str, unicode], bool, bool) -> Union[str, unicode] if url in ("@id", "@type"): return url @@ -127,7 +127,7 @@ def expand_url(self, url, base_url, scoped=False, vocab_term=False): if split.scheme or url.startswith("$(") or url.startswith("${"): pass - elif scoped and not split.fragment: + elif scoped_id and not split.fragment: splitbase = urlparse.urlsplit(base_url) frg = "" if splitbase.fragment: @@ -136,6 +136,8 @@ def expand_url(self, url, base_url, scoped=False, vocab_term=False): frg = split.path url = urlparse.urlunsplit( (splitbase.scheme, splitbase.netloc, splitbase.path, splitbase.query, frg)) + elif scoped_ref is not None and not split.fragment: + pass else: url = urlparse.urljoin(base_url, url) @@ -219,6 +221,8 @@ def add_context(self, newcontext, baseuri=""): elif isinstance(value, dict) and value.get("@type") == "@vocab": self.url_fields.add(key) self.vocab_fields.add(key) + if "refScope" in value: + self.scoped_ref_fields[key] = value["refScope"] if value.get("typeDSL"): self.type_dsl_fields.add(key) if isinstance(value, dict) and value.get("noLinkCheck"): @@ -236,7 +240,7 @@ def add_context(self, newcontext, baseuri=""): self.vocab[key] = value for k, v in self.vocab.items(): - self.rvocab[self.expand_url(v, "", scoped=False)] = k + self.rvocab[self.expand_url(v, "", scoped_id=False)] = k _logger.debug("identifiers is %s", self.identifiers) _logger.debug("identity_links is %s", self.identity_links) @@ -282,7 +286,7 @@ def resolve_ref(self, ref, base_url=None, checklinks=True): if not isinstance(ref, (str, unicode)): raise ValueError("Must be string: `%s`" % str(ref)) - url = self.expand_url(ref, base_url, scoped=(obj is not None)) + url = self.expand_url(ref, base_url, scoped_id=(obj is not None)) # Has this reference been loaded already? if url in self.idx: @@ -364,7 +368,6 @@ def _type_dsl(self, t): "items": r} if m.group(3): r = ["null", r] - print t, "DSL to", r return r def _resolve_type_dsl(self, document, loader): @@ -391,7 +394,7 @@ def _resolve_identifier(self, document, loader, base_url): if identifer in document: if isinstance(document[identifer], basestring): document[identifer] = loader.expand_url( - document[identifer], base_url, scoped=True) + document[identifer], base_url, scoped_id=True) if document[identifer] not in loader.idx or isinstance(loader.idx[document[identifer]], basestring): loader.idx[document[identifer]] = document base_url = document[identifer] @@ -408,7 +411,7 @@ def _resolve_identity(self, document, loader, base_url): for n, v in enumerate(document[identifer]): if isinstance(document[identifer][n], basestring): document[identifer][n] = loader.expand_url( - document[identifer][n], base_url, scoped=True) + document[identifer][n], base_url, scoped_id=True) if document[identifer][n] not in loader.idx: loader.idx[document[identifer][ n]] = document[identifer][n] @@ -416,7 +419,7 @@ def _resolve_identity(self, document, loader, base_url): def _normalize_fields(self, document, loader): # Normalize fields which are prefixed or full URIn to vocabulary terms for d in document: - d2 = loader.expand_url(d, "", scoped=False, vocab_term=True) + d2 = loader.expand_url(d, "", scoped_id=False, vocab_term=True) if d != d2: document[d2] = document[d] del document[d] @@ -424,17 +427,18 @@ def _normalize_fields(self, document, loader): def _resolve_uris(self, document, loader, base_url): # Resolve remaining URLs based on document base for d in loader.url_fields: - if d in self.scoped_ref_fields: - continue if d in document: if isinstance(document[d], basestring): document[d] = loader.expand_url( - document[d], base_url, scoped=False, vocab_term=(d in loader.vocab_fields)) + document[d], base_url, scoped_id=False, + vocab_term=(d in loader.vocab_fields), + scoped_ref=self.scoped_ref_fields.get(d)) elif isinstance(document[d], list): document[d] = [ loader.expand_url( - url, base_url, scoped=False, - vocab_term=(d in loader.vocab_fields)) + url, base_url, scoped_id=False, + vocab_term=(d in loader.vocab_fields), + scoped_ref=self.scoped_ref_fields.get(d)) if isinstance(url, (str, unicode)) else url for url in document[d]] @@ -531,7 +535,7 @@ def resolve_all(self, document, base_url, file_base=None, checklinks=True): if identifer in metadata: if isinstance(metadata[identifer], (str, unicode)): metadata[identifer] = loader.expand_url( - metadata[identifer], base_url, scoped=True) + metadata[identifer], base_url, scoped_id=True) loader.idx[metadata[identifer]] = document if checklinks: @@ -598,6 +602,28 @@ def check_file(self, fn): # type: (Union[str, unicode]) -> bool FieldType = TypeVar('FieldType', unicode, List[str], Dict[str, Any]) + def validate_scoped(self, field, link, docid): + split = urlparse.urlsplit(docid) + sp = split.fragment.split("/") + n = self.scoped_ref_fields[field] + while n > 0 and len(sp) > 0: + sp.pop() + n -= 1 + tried = [] + while True: + sp.append(str(link)) + url = urlparse.urlunsplit( + (split.scheme, split.netloc, split.path, split.query, "/".join(sp))) + tried.append(url) + if url in self.idx: + return url + sp.pop() + if len(sp) == 0: + break + sp.pop() + raise validate.ValidationException( + "Field `%s` contains undefined reference to `%s`, tried %s" % (field, link, tried)) + def validate_link(self, field, link, docid): # type: (AnyStr, FieldType, AnyStr) -> FieldType if field in self.nolinkcheck: @@ -605,29 +631,14 @@ def validate_link(self, field, link, docid): if isinstance(link, (str, unicode)): if field in self.vocab_fields: if link not in self.vocab and link not in self.idx and link not in self.rvocab: - if not self.check_file(link): + if field in self.scoped_ref_fields: + return self.validate_scoped(field, link, docid) + elif not self.check_file(link): raise validate.ValidationException( "Field `%s` contains undefined reference to `%s`" % (field, link)) elif link not in self.idx and link not in self.rvocab: if field in self.scoped_ref_fields: - split = urlparse.urlsplit(docid) - sp = split.fragment.split("/") - n = self.scoped_ref_fields[field] - while n > 0 and len(sp) > 0: - sp.pop() - n -= 1 - while True: - sp.append(str(link)) - url = urlparse.urlunsplit( - (split.scheme, split.netloc, split.path, split.query, "/".join(sp))) - if url in self.idx: - return url - sp.pop() - if len(sp) == 0: - break - sp.pop() - raise validate.ValidationException( - "Field `%s` contains undefined reference to `%s`" % (field, link)) + return self.validate_scoped(field, link, docid) elif not self.check_file(link): raise validate.ValidationException( "Field `%s` contains undefined reference to `%s`" % (field, link)) diff --git a/schema_salad/schema.py b/schema_salad/schema.py index 9be8ac718..067538f72 100644 --- a/schema_salad/schema.py +++ b/schema_salad/schema.py @@ -93,7 +93,8 @@ def get_metaschema(): "enum": "https://w3id.org/cwl/salad#enum", "extends": { "@id": "https://w3id.org/cwl/salad#extends", - "@type": "@id" + "@type": "@id", + "refScope": 0 }, "fields": "sld:fields", "float": "http://www.w3.org/2001/XMLSchema#float", @@ -101,7 +102,8 @@ def get_metaschema(): "int": "http://www.w3.org/2001/XMLSchema#int", "items": { "@id": "https://w3id.org/cwl/salad#items", - "@type": "@vocab" + "@type": "@vocab", + "refScope": 0 }, "jsonldPredicate": "sld:jsonldPredicate", "long": "http://www.w3.org/2001/XMLSchema#long", @@ -133,7 +135,8 @@ def get_metaschema(): "type": { "@id": "https://w3id.org/cwl/salad#type", "@type": "@vocab", - "typeDSL": True + "typeDSL": True, + "refScope": 0 }, "typeDSL": "https://w3id.org/cwl/salad#JsonldPredicate/typeDSL", "xsd": "http://www.w3.org/2001/XMLSchema#" diff --git a/tests/test_examples.py b/tests/test_examples.py index 2e847d0a2..8a657f445 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -139,7 +139,8 @@ def test_scoped_ref(self): ra, _ = ldr.resolve_all({ "inputs": { - "inp": "string" + "inp": "string", + "inp2": "string" }, "outputs": { "out": { @@ -150,7 +151,9 @@ def test_scoped_ref(self): "steps": { "step1": { "in": { - "inp": "inp" + "inp": "inp", + "inp2": "#inp2", + "inp3": ["inp", "inp2"] }, "out": ["out"], "scatter": "inp" @@ -181,6 +184,9 @@ def test_scoped_ref(self): 'in': [{ 'id': 'http://example2.com/#step1/inp', 'source': 'http://example2.com/#inp' + }, { + 'id': 'http://example2.com/#step1/inp2', + 'source': 'http://example2.com/#inp' }], "out": ["http://example2.com/#step1/out"], }, { @@ -223,13 +229,16 @@ def test_typedsl_ref(self): }) ra, _ = ldr.resolve_all({"type": "File"}, "") - print ra + self.assertEqual({'type': 'File'}, ra) + ra, _ = ldr.resolve_all({"type": "File?"}, "") - print ra + self.assertEqual({'type': ['null', 'File']}, ra) + ra, _ = ldr.resolve_all({"type": "File[]"}, "") - print ra + self.assertEqual({'type': {'items': 'File', 'type': 'array'}}, ra) + ra, _ = ldr.resolve_all({"type": "File[]?"}, "") - print ra + self.assertEqual({'type': ['null', {'items': 'File', 'type': 'array'}]}, ra) if __name__ == '__main__': unittest.main()