encoding/jsonschema: implement MapURL

This allows the caller of `encoding/jsonschema.Extract` to choose how to map schema IDs to import paths on an arbitrary basis. Signed-off-by: Roger Peppe <[email protected]> Change-Id: If04075a404e91a3694ca158a31c6d039a531954b Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1199210 Reviewed-by: Daniel Martí <[email protected]> TryBot-Result: CUEcueckoo <[email protected]> Unity-Result: CUE porcuepine <[email protected]>
cue-lang · Aug 13, 2024 · bddcb54 · bddcb54
1 parent c683420
commit bddcb54
Show file tree

Hide file tree

Showing 4 changed files with 131 additions and 55 deletions.
diff --git a/encoding/jsonschema/constraints.go b/encoding/jsonschema/constraints.go
@@ -16,7 +16,6 @@ package jsonschema
 
 import (
 	"math/big"
-	"path"
 	"regexp"
 
 	"cuelang.org/go/cue"
@@ -261,14 +260,12 @@ var constraints = []*constraint{
 
 		u := s.resolveURI(n)
 
-		if u.Fragment != "" && !path.IsAbs(u.Fragment) {
-			s.addErr(errors.Newf(n.Pos(), "anchors (%s) not supported", u.Fragment))
-			// TODO: support anchors
+		fragmentParts, err := splitFragment(u)
+		if err != nil {
+			s.addErr(errors.Newf(n.Pos(), "%v", err))
 			return
 		}
-
-		expr := s.makeCUERef(n, u)
-
+		expr := s.makeCUERef(n, u, fragmentParts)
 		if expr == nil {
 			expr = &ast.BadExpr{From: n.Pos()}
 		}

diff --git a/encoding/jsonschema/decode_test.go b/encoding/jsonschema/decode_test.go
@@ -16,9 +16,12 @@ package jsonschema_test
 
 import (
 	"bytes"
+	"fmt"
+	"net/url"
 	"path"
 	"testing"
 
+	"github.com/go-quicktest/qt"
 	"golang.org/x/tools/txtar"
 
 	"cuelang.org/go/cue"
@@ -104,6 +107,56 @@ func TestDecode(t *testing.T) {
 	})
 }
 
+func TestMapURL(t *testing.T) {
+	v := cuecontext.New().CompileString(`
+$schema: "xxx"
+type: "object"
+properties: x: $ref: "https://something.test/foo#/definitions/blah"
+`)
+	var calls []string
+	expr, err := jsonschema.Extract(v, &jsonschema.Config{
+		MapURL: func(u *url.URL) (string, error) {
+			calls = append(calls, u.String())
+			return "other.test/something:blah", nil
+		},
+	})
+	qt.Assert(t, qt.IsNil(err))
+	b, err := format.Node(expr, format.Simplify())
+	if err != nil {
+		t.Fatal(errors.Details(err, nil))
+	}
+	qt.Assert(t, qt.DeepEquals(calls, []string{"https://something.test/foo"}))
+	qt.Assert(t, qt.Equals(string(b), `
+import "other.test/something:blah"
+
+@jsonschema(schema="xxx")
+x?: blah.#blah
+...
+`[1:]))
+}
+
+func TestMapURLErrors(t *testing.T) {
+	v := cuecontext.New().CompileString(`
+$schema: "xxx"
+type: "object"
+properties: {
+	x: $ref: "https://something.test/foo#/definitions/x"
+	y: $ref: "https://something.test/foo#/definitions/y"
+}
+`, cue.Filename("foo.cue"))
+	_, err := jsonschema.Extract(v, &jsonschema.Config{
+		MapURL: func(u *url.URL) (string, error) {
+			return "", fmt.Errorf("some error")
+		},
+	})
+	qt.Assert(t, qt.Equals(errors.Details(err, nil), `
+cannot determine import path from URL "https://something.test/foo": some error:
+    foo.cue:5:5
+cannot determine import path from URL "https://something.test/foo": some error:
+    foo.cue:6:5
+`[1:]))
+}
+
 func TestX(t *testing.T) {
 	t.Skip()
 	data := `

diff --git a/encoding/jsonschema/jsonschema.go b/encoding/jsonschema/jsonschema.go
@@ -32,6 +32,8 @@
 package jsonschema
 
 import (
+	"net/url"
+
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/ast"
 	"cuelang.org/go/cue/token"
@@ -42,6 +44,11 @@ import (
 // The generated CUE schema is guaranteed to deem valid any value that is
 // a valid instance of the source JSON schema.
 func Extract(data cue.InstanceOrValue, cfg *Config) (f *ast.File, err error) {
+	if cfg.MapURL == nil {
+		cfg1 := *cfg
+		cfg = &cfg1
+		cfg1.MapURL = DefaultMapURL
+	}
 	d := &decoder{cfg: cfg}
 
 	f = d.decode(data.Value())
@@ -76,6 +83,11 @@ type Config struct {
 	//    {"$defs", foo}         {#foo} or {#, foo}
 	Map func(pos token.Pos, path []string) ([]ast.Label, error)
 
+	// MapURL maps a URL reference as found in $ref to
+	// an import path for a package.
+	// If this is nil, [DefaultMapURL] will be used.
+	MapURL func(u *url.URL) (importPath string, err error)
+
 	// TODO: configurability to make it compatible with OpenAPI, such as
 	// - locations of definitions: #/components/schemas, for instance.
 	// - selection and definition of formats

diff --git a/encoding/jsonschema/ref.go b/encoding/jsonschema/ref.go
@@ -15,6 +15,7 @@
 package jsonschema
 
 import (
+	"fmt"
 	"net/url"
 	"path"
 	"strconv"
@@ -25,6 +26,7 @@ import (
 	"cuelang.org/go/cue/errors"
 	"cuelang.org/go/cue/token"
 	"cuelang.org/go/internal"
+	"cuelang.org/go/mod/module"
 )
 
 func (d *decoder) parseRef(p token.Pos, str string) []string {
@@ -35,7 +37,7 @@ func (d *decoder) parseRef(p token.Pos, str string) []string {
 	}
 
 	if u.Host != "" || u.Path != "" {
-		d.addErr(errors.Newf(p, "external references (%s) not supported", str))
+		d.addErr(errors.Newf(p, "external references (%s) not supported in Root", str))
 		// TODO: handle
 		//    host:
 		//      If the host corresponds to a package known to cue,
@@ -47,18 +49,12 @@ func (d *decoder) parseRef(p token.Pos, str string) []string {
 		//      Look up on file system or relatively to authority location.
 		return nil
 	}
-
-	if !path.IsAbs(u.Fragment) {
-		d.addErr(errors.Newf(p, "anchors (%s) not supported", u.Fragment))
-		// TODO: support anchors
+	fragmentParts, err := splitFragment(u)
+	if err != nil {
+		d.addErr(errors.Newf(p, "%v", err))
 		return nil
 	}
-
-	// NOTE: Go bug?: url.URL has no raw representation of the fragment. This
-	// means that %2F gets translated to `/` before it can be split. This, in
-	// turn, means that field names cannot have a `/` as name.
-
-	return splitFragment(u)
+	return fragmentParts
 }
 
 // resolveURI parses a URI from n and resolves it in the current context.
@@ -99,15 +95,13 @@ const topSchema = "_schema"
 // The returned identifier (or first expression in a selection chain), is
 // hardwired to point to the resolved value. This will allow astutil.Sanitize
 // to automatically unshadow any shadowed variables.
-func (s *state) makeCUERef(n cue.Value, u *url.URL) ast.Expr {
-	a := splitFragment(u)
-
+func (s *state) makeCUERef(n cue.Value, u *url.URL, fragmentParts []string) (_e ast.Expr) {
 	switch fn := s.cfg.Map; {
 	case fn != nil:
 		// TODO: This block is only used in case s.cfg.Map is set, which is
 		// currently only used for OpenAPI. Handling should be brought more in
 		// line with JSON schema.
-		a, err := fn(n.Pos(), a)
+		a, err := fn(n.Pos(), fragmentParts)
 		if err != nil {
 			s.addErr(errors.Newf(n.Pos(), "invalid reference %q: %v", u, err))
 			return nil
@@ -141,7 +135,7 @@ func (s *state) makeCUERef(n cue.Value, u *url.URL) ast.Expr {
 			switch {
 			case u.Host == "" && u.Path == "",
 				s.id != nil && s.id.Host == u.Host && s.id.Path == u.Path:
-				if len(a) == 0 {
+				if len(fragmentParts) == 0 {
 					// refers to the top of the file. We will allow this by
 					// creating a helper schema as such:
 					//   _schema: {...}
@@ -155,33 +149,27 @@ func (s *state) makeCUERef(n cue.Value, u *url.URL) ast.Expr {
 					return ident
 				}
 
-				ident, a = s.getNextIdent(n, a)
+				ident, fragmentParts = s.getNextIdent(n, fragmentParts)
 
 			case u.Host != "":
 				// Reference not found within scope. Create an import reference.
 
-				// TODO: allow the configuration to specify a map from
-				// URI domain+paths to CUE packages.
-
 				// TODO: currently only $ids that are in scope can be
 				// referenced. We could consider doing an extra pass to record
 				// all '$id's in a file to be able to link to them even if they
 				// are not in scope.
-				p := u.Path
-
-				base := path.Base(p)
-				if !ast.IsValidIdent(base) {
-					base = strings.TrimSuffix(base, ".json")
-					if !ast.IsValidIdent(base) {
-						// Find something more clever to do there. For now just
-						// pick "schema" as the package name.
-						base = "schema"
-					}
-					p += ":" + base
+				importPath, err := s.cfg.MapURL(u)
+				if err != nil {
+					s.errf(n, "cannot determine import path from URL %q: %v", u, err)
+					return nil
 				}
-
-				ident = ast.NewIdent(base)
-				ident.Node = &ast.ImportSpec{Path: ast.NewString(u.Host + p)}
+				ip := module.ParseImportPath(importPath)
+				if ip.Qualifier == "" {
+					s.errf(n, "cannot determine package name from import path %q", importPath)
+					return nil
+				}
+				ident = ast.NewIdent(ip.Qualifier)
+				ident.Node = &ast.ImportSpec{Path: ast.NewString(importPath)}
 
 			default:
 				// Just a path, not sure what that means.
@@ -196,7 +184,7 @@ func (s *state) makeCUERef(n cue.Value, u *url.URL) ast.Expr {
 		}
 
 		if s.id.Host == u.Host && s.id.Path == u.Path {
-			if len(a) == 0 {
+			if len(fragmentParts) == 0 {
 				if len(s.idRef) == 0 {
 					// This is a reference to either root or a schema for which
 					// we do not yet support references. See Issue #386.
@@ -232,13 +220,13 @@ func (s *state) makeCUERef(n cue.Value, u *url.URL) ast.Expr {
 				}
 				return newSel(e, s.idRef[1])
 			}
-			ident, a = s.getNextIdent(n, a)
+			ident, fragmentParts = s.getNextIdent(n, fragmentParts)
 			ident.Node = s.obj
 			break
 		}
 	}
 
-	return s.newSel(ident, n, a)
+	return s.newSel(ident, n, fragmentParts)
 }
 
 // getNextSelector translates a JSON Reference path into a CUE path by consuming
@@ -377,20 +365,27 @@ func (s *state) linkReferences() {
 	}
 }
 
-// splitFragment splits the fragment part of a URI into path components. The
-// result may be an empty slice.
+// splitFragment splits the fragment part of a URI into path components
+// and removes the fragment part from u.
+// The result may be an empty slice.
 //
-// TODO: this requires RawFragment introduced in go1.15 to function properly.
-// As for now, CUE still uses go1.12.
-func splitFragment(u *url.URL) []string {
-	if u.Fragment == "" {
-		return nil
+// TODO: use u.RawFragment so that we can accept field names
+// that contain `/` characters.
+func splitFragment(u *url.URL) ([]string, error) {
+	frag := u.EscapedFragment()
+	if frag == "" {
+		return nil, nil
 	}
-	s := strings.TrimRight(u.Fragment[1:], "/")
-	if s == "" {
-		return nil
+	if !strings.HasPrefix(frag, "/") {
+		return nil, fmt.Errorf("anchors (%s) not supported", frag)
+	}
+	u.Fragment = ""
+	u.RawFragment = ""
+
+	if s := strings.TrimRight(frag[1:], "/"); s != "" {
+		return strings.Split(s, "/"), nil
 	}
-	return strings.Split(s, "/")
+	return nil, nil
 }
 
 func (d *decoder) mapRef(p token.Pos, str string, ref []string) []ast.Label {
@@ -438,3 +433,22 @@ func jsonSchemaRef(p token.Pos, a []string) ([]ast.Label, error) {
 	}
 	return []ast.Label{ast.NewIdent(rootDefs), ast.NewString(name)}, nil
 }
+
+// DefaultMapURL implements the default schema ID to import
+// path mapping. It trims off any ".json" suffix and uses the
+// package name "schema" if the final component of the path
+// isn't a valid CUE identifier.
+func DefaultMapURL(u *url.URL) (importPath string, err error) {
+	p := u.Path
+	base := path.Base(p)
+	if !ast.IsValidIdent(base) {
+		base = strings.TrimSuffix(base, ".json")
+		if !ast.IsValidIdent(base) {
+			// Find something more clever to do there. For now just
+			// pick "schema" as the package name.
+			base = "schema"
+		}
+		p += ":" + base
+	}
+	return u.Host + p, nil
+}