encoding/jsonschema: add external test suite

This adds test data acquired from
github.com/json-schema-org/JSON-Schema-Test-Suite and uses it to test
CUE's encoding/jsonschema package.

Clearly, there are many tests and schemas that fail, so we need to avoid
those failures causing the CUE CI tests to fail. We could maintain an
auxilliary data structure to record which tests are expected to
pass/fail, but that would be hard to understand, because each failure is
only understandable in the context of the test data that produced it.

So instead, we add the auxilliary data directly inside the JSON data
directly, using the `CUE_UPDATE` convention to enable updating that
information. Specifically each schema and each test can be associated
with a `skip` field that causes a test failure on that schema or that
test to be ignored, and describes the reason for the failure.

The `vendor-external` script can be used to pull in updated tests from
the external repository.

The `teststats.go` program can be used to show information on which
tests pass and fail.

At the time of writing, the summary stats are as follows:

	schema extract (pass / total): 971 / 1637 = 59.3%
	tests (pass / total): 3032 / 7175 = 42.3%
	tests on extracted schemas (pass / total): 3032 / 3542 = 85.6%

Also in passing add some copyright headers to some files that lacked them.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: I653201803df8a9165671bd79929e12f37e549258
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200255
Reviewed-by: Daniel Martí <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>

encoding/jsonschema/external_test.go

@@ -0,0 +1,202 @@
+// Copyright 2024 CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jsonschema_test
+
+import (
+	"bytes"
+	stdjson "encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/go-quicktest/qt"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/cuecontext"
+	"cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/format"
+	"cuelang.org/go/encoding/json"
+	"cuelang.org/go/encoding/jsonschema"
+	"cuelang.org/go/encoding/jsonschema/internal/externaltest"
+	"cuelang.org/go/internal/cuetest"
+)
+
+// TestExternal runs the externally defined JSON Schema test suite,
+// as defined in https://github.com/json-schema-org/JSON-Schema-Test-Suite.
+func TestExternal(t *testing.T) {
+	tests, err := externaltest.ReadTestDir("testdata/external")
+	qt.Assert(t, qt.IsNil(err))
+
+	// Group the tests under a single subtest so that we can use
+	// t.Parallel and still guarantee that all tests have completed
+	// by the end.
+	t.Run("tests", func(t *testing.T) {
+		// Run tests in deterministic order so we get some consistency between runs.
+		for _, filename := range sortedKeys(tests) {
+			schemas := tests[filename]
+			t.Run(testName(filename), func(t *testing.T) {
+				for _, s := range schemas {
+					t.Run(testName(s.Description), func(t *testing.T) {
+						runExternalSchemaTests(t, filename, s)
+					})
+				}
+			})
+		}
+	})
+	if !cuetest.UpdateGoldenFiles {
+		return
+	}
+	for filename, schemas := range tests {
+		filename = filepath.Join("testdata/external", filename)
+		data, err := stdjson.MarshalIndent(schemas, "", "\t")
+		qt.Assert(t, qt.IsNil(err))
+		data = append(data, '\n')
+		oldData, err := os.ReadFile(filename)
+		qt.Assert(t, qt.IsNil(err))
+		if bytes.Equal(oldData, data) {
+			continue
+		}
+		err = os.WriteFile(filename, data, 0o666)
+		qt.Assert(t, qt.IsNil(err))
+	}
+}
+
+func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schema) {
+	t.Logf("file %v", path.Join("testdata/external", filename))
+	ctx := cuecontext.New()
+	jsonAST, err := json.Extract("schema.json", s.Schema)
+	qt.Assert(t, qt.IsNil(err))
+	jsonValue := ctx.BuildExpr(jsonAST)
+	qt.Assert(t, qt.IsNil(jsonValue.Err()))
+	versStr, _, _ := strings.Cut(strings.TrimPrefix(filename, "tests/"), "/")
+	vers, ok := extVersionToVersion[versStr]
+	if !ok {
+		t.Fatalf("unknown JSON schema version for file %q", filename)
+	}
+	if vers == jsonschema.VersionUnknown {
+		t.Skipf("skipping test for unknown schema version %v", versStr)
+	}
+	schemaAST, extractErr := jsonschema.Extract(jsonValue, &jsonschema.Config{
+		Strict:         true,
+		DefaultVersion: vers,
+	})
+	var schemaValue cue.Value
+	if extractErr == nil {
+		// Round-trip via bytes because that's what will usually happen
+		// to the generated schema.
+		b, err := format.Node(schemaAST, format.Simplify())
+		qt.Assert(t, qt.IsNil(err))
+		schemaValue = ctx.CompileBytes(b, cue.Filename("generated.cue"))
+		if err := schemaValue.Err(); err != nil {
+			extractErr = fmt.Errorf("cannot compile resulting schema: %v", errors.Details(err, nil))
+		}
+	}
+
+	if extractErr != nil {
+		if cuetest.UpdateGoldenFiles {
+			s.Skip = fmt.Sprintf("extract error: %v", extractErr)
+			for _, t := range s.Tests {
+				t.Skip = "could not compile schema"
+			}
+			return
+		}
+		if s.Skip != "" {
+			t.SkipNow()
+		}
+		t.Fatalf("extract error: %v", extractErr)
+	}
+	if s.Skip != "" {
+		t.Errorf("unexpected test success on skipped test")
+	}
+
+	for _, test := range s.Tests {
+		t.Run(testName(test.Description), func(t *testing.T) {
+			instAST, err := json.Extract("instance.json", test.Data)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			qt.Assert(t, qt.IsNil(err), qt.Commentf("test data: %q; details: %v", test.Data, errors.Details(err, nil)))
+
+			instValue := ctx.BuildExpr(instAST)
+			qt.Assert(t, qt.IsNil(instValue.Err()))
+			err = instValue.Unify(schemaValue).Err()
+			if test.Valid {
+				if cuetest.UpdateGoldenFiles {
+					if err == nil {
+						test.Skip = ""
+					} else {
+						test.Skip = errors.Details(err, nil)
+					}
+					return
+				}
+				if err != nil {
+					if test.Skip != "" {
+						t.Skipf("skipping due to known error: %v", test.Skip)
+					}
+					t.Fatalf("error: %v", errors.Details(err, nil))
+				} else if test.Skip != "" {
+					t.Fatalf("unexpectedly more correct behavior (test success) on skipped test")
+				}
+			} else {
+				if cuetest.UpdateGoldenFiles {
+					if err != nil {
+						test.Skip = ""
+					} else {
+						test.Skip = "unexpected success"
+					}
+					return
+				}
+				if err == nil {
+					if test.Skip != "" {
+						t.SkipNow()
+					}
+					t.Fatal("unexpected success")
+				} else if test.Skip != "" {
+					t.Fatalf("unexpectedly more correct behavior (test failure) on skipped test")
+				}
+			}
+		})
+	}
+}
+
+// testName returns a test name that doesn't contain any
+// slashes because slashes muck with matching.
+func testName(s string) string {
+	return strings.ReplaceAll(s, "/", "__")
+}
+
+var extVersionToVersion = map[string]jsonschema.Version{
+	"draft3":       jsonschema.VersionUnknown,
+	"draft4":       jsonschema.VersionDraft4,
+	"draft6":       jsonschema.VersionDraft6,
+	"draft7":       jsonschema.VersionDraft7,
+	"draft2019-09": jsonschema.VersionDraft2019_09,
+	"draft2020-12": jsonschema.VersionDraft2020_12,
+	"draft-next":   jsonschema.VersionUnknown,
+}
+
+func sortedKeys[T any](m map[string]T) []string {
+	ks := make([]string, 0, len(m))
+	for k := range m {
+		ks = append(ks, k)
+	}
+	sort.Strings(ks)
+	return ks
+}

encoding/jsonschema/internal/externaltest/tests.go

@@ -0,0 +1,65 @@
+package externaltest
+
+import (
+	"bytes"
+	stdjson "encoding/json"
+	"os"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/cuecontext"
+	"cuelang.org/go/cue/interpreter/embed"
+	"cuelang.org/go/cue/load"
+)
+
+type Schema struct {
+	Description string             `json:"description"`
+	Comment     string             `json:"comment,omitempty"`
+	Schema      stdjson.RawMessage `json:"schema"`
+	Skip        string             `json:"skip,omitempty"`
+	Tests       []*Test            `json:"tests"`
+}
+
+type Test struct {
+	Description string             `json:"description"`
+	Comment     string             `json:"comment,omitempty"`
+	Data        stdjson.RawMessage `json:"data"`
+	Valid       bool               `json:"valid"`
+	Skip        string             `json:"skip,omitempty"`
+}
+
+func ReadTestDir(dir string) (tests map[string][]*Schema, err error) {
+	os.Setenv("CUE_EXPERIMENT", "embed")
+	inst := load.Instances([]string{"."}, &load.Config{
+		Dir: dir,
+	})[0]
+	if err != nil {
+		return nil, err
+	}
+	ctx := cuecontext.New(cuecontext.Interpreter(embed.New()))
+	instVal := ctx.BuildInstance(inst)
+	if err := instVal.Err(); err != nil {
+		return nil, err
+	}
+	val := instVal.LookupPath(cue.MakePath(cue.Str("allTests")))
+	if err := val.Err(); err != nil {
+		return nil, err
+	}
+	if err := val.Decode(&tests); err != nil {
+		return nil, err
+	}
+	// Fix up the raw JSON data to avoid running into some decode issues.
+	for _, schemas := range tests {
+		for _, schema := range schemas {
+			for _, test := range schema.Tests {
+				if len(test.Data) == 0 {
+					// See https://github.com/cue-lang/cue/issues/3397
+					test.Data = []byte("null")
+					continue
+				}
+				// See https://github.com/cue-lang/cue/issues/3398
+				test.Data = bytes.ReplaceAll(test.Data, []byte("\ufeff"), []byte(`\ufeff`))
+			}
+		}
+	}
+	return tests, nil
+}

encoding/jsonschema/testdata/external/config.cue

@@ -0,0 +1,39 @@
+@extern(embed)
+
+package external
+
+// TODO use recursive globbing when it's available.
+
+allTests: _	@embed(glob=tests/*/*.json)
+allTests: _	@embed(glob=tests/*/*/*.json)
+allTests: _	@embed(glob=tests/*/*/*/*.json)
+
+allTests: [_]: [... #Schema]
+#Schema: {
+	description!: string
+	comment?: string
+	specification?: _
+	schema!: _
+	tests!: [... #Test]
+
+	// skip is not part of the orginal test data, but
+	// is inserted by our test logic (when CUE_UPDATE=1)
+	// to indicate which tests are passing and which
+	// are failing. The text indicates some reason as to
+	// why the schema is skipped.
+	skip?: string
+}
+
+#Test: {
+	description!: string
+	comment?: string
+	data!: _
+	valid!: bool
+
+	// skip is not part of the orginal test data, but
+	// is inserted by our test logic (when CUE_UPDATE=1)
+	// to indicate which tests are passing and which
+	// are failing. The text indicates some reason as to
+	// why the test is skipped.
+	skip?: string
+}