Skip to content

Commit

Permalink
encoding/jsonschema: run external tests in matrix
Browse files Browse the repository at this point in the history 
This change now runs the external tests in both the v2 and v3 evaluators
and updates the respective test skip fields accordingly.

There is a lot of duplication, naturally, but it seems better to keep
the testing code simpler by having independently updated fields than to
try to combine them when they're the same. It'll be straightforward to
change the `teststats` helper program to summarise differences between
v2 and v3 in a human-readable fashion as required.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: Id21625d2b3d3e442a64f91b1fac39b1fe418b2fd
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200520
Unity-Result: CUE porcuepine <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>
  • Loading branch information
@rogpeppe
rogpeppe committed Sep 3, 2024
1 parent 9959a3e commit e2685dd
Show file tree
Hide file tree
Showing 257 changed files with 9,759 additions and 4,846 deletions.
44 changes: 25 additions & 19 deletions encoding/jsonschema/external_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ import (
"github.com/go-quicktest/qt"

"cuelang.org/go/cue"
"cuelang.org/go/cue/cuecontext"
"cuelang.org/go/cue/errors"
"cuelang.org/go/cue/format"
"cuelang.org/go/cue/token"
"cuelang.org/go/encoding/json"
"cuelang.org/go/encoding/jsonschema"
"cuelang.org/go/encoding/jsonschema/internal/externaltest"
"cuelang.org/go/internal/cuetdtest"
"cuelang.org/go/internal/cuetest"
)

Expand All @@ -52,14 +52,14 @@ func TestExternal(t *testing.T) {
// Group the tests under a single subtest so that we can use
// t.Parallel and still guarantee that all tests have completed
// by the end.
t.Run("tests", func(t *testing.T) {
cuetdtest.SmallMatrix.Run(t, "tests", func(t *testing.T, m *cuetdtest.M) {
// Run tests in deterministic order so we get some consistency between runs.
for _, filename := range sortedKeys(tests) {
schemas := tests[filename]
t.Run(testName(filename), func(t *testing.T) {
for _, s := range schemas {
t.Run(testName(s.Description), func(t *testing.T) {
runExternalSchemaTests(t, filename, s)
runExternalSchemaTests(t, m, filename, s)
})
}
})
Expand All @@ -75,9 +75,9 @@ func TestExternal(t *testing.T) {
qt.Assert(t, qt.IsNil(err))
}

func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schema) {
func runExternalSchemaTests(t *testing.T, m *cuetdtest.M, filename string, s *externaltest.Schema) {
t.Logf("file %v", path.Join("testdata/external", filename))
ctx := cuecontext.New()
ctx := m.CueContext()
jsonAST, err := json.Extract("schema.json", s.Schema)
qt.Assert(t, qt.IsNil(err))
jsonValue := ctx.BuildExpr(jsonAST)
Expand Down Expand Up @@ -112,13 +112,13 @@ func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schem
t.Logf("txtar:\n%s", schemaFailureTxtar(s))
for _, test := range s.Tests {
t.Run("", func(t *testing.T) {
testFailed(t, &test.Skip, test, "could not compile schema")
testFailed(t, m, &test.Skip, test, "could not compile schema")
})
}
testFailed(t, &s.Skip, s, fmt.Sprintf("extract error: %v", extractErr))
testFailed(t, m, &s.Skip, s, fmt.Sprintf("extract error: %v", extractErr))
return
}
testSucceeded(t, &s.Skip, s)
testSucceeded(t, m, &s.Skip, s)

for _, test := range s.Tests {
t.Run(testName(test.Description), func(t *testing.T) {
Expand All @@ -140,15 +140,15 @@ func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schem
err = instValue.Unify(schemaValue).Err()
if test.Valid {
if err != nil {
testFailed(t, &test.Skip, test, errors.Details(err, nil))
testFailed(t, m, &test.Skip, test, errors.Details(err, nil))
} else {
testSucceeded(t, &test.Skip, test)
testSucceeded(t, m, &test.Skip, test)
}
} else {
if err == nil {
testFailed(t, &test.Skip, test, "unexpected success")
testFailed(t, m, &test.Skip, test, "unexpected success")
} else {
testSucceeded(t, &test.Skip, test)
testSucceeded(t, m, &test.Skip, test)
}
}
})
Expand Down Expand Up @@ -203,28 +203,34 @@ func testName(s string) string {
// testFailed marks the current test as failed with the
// given error message, and updates the
// skip field pointed to by skipField if necessary.
func testFailed(t *testing.T, skipField *externaltest.Skip, p positioner, errStr string) {
func testFailed(t *testing.T, m *cuetdtest.M, skipField *externaltest.Skip, p positioner, errStr string) {
if cuetest.UpdateGoldenFiles {
if *skipField == nil && !allowRegressions() {
t.Fatalf("test regression; was succeeding, now failing: %v", errStr)
}
*skipField = externaltest.Skip{"v2": errStr}
if *skipField == nil {
*skipField = make(externaltest.Skip)
}
(*skipField)[m.Name()] = errStr
return
}
if *skipField != nil {
t.Skipf("skipping due to known error: %v", *skipField)
if reason := (*skipField)[m.Name()]; reason != "" {
t.Skipf("skipping due to known error: %v", reason)
}
t.Fatal(errStr)
}

// testFails marks the current test as succeeded and updates the
// skip field pointed to by skipField if necessary.
func testSucceeded(t *testing.T, skipField *externaltest.Skip, p positioner) {
func testSucceeded(t *testing.T, m *cuetdtest.M, skipField *externaltest.Skip, p positioner) {
if cuetest.UpdateGoldenFiles {
*skipField = nil
delete(*skipField, m.Name())
if len(*skipField) == 0 {
*skipField = nil
}
return
}
if *skipField != nil {
if reason := (*skipField)[m.Name()]; reason != "" {
t.Fatalf("unexpectedly more correct behavior (test success) on skipped test")
}
}
Expand Down
15 changes: 10 additions & 5 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/additionalItems.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@
"data": [],
"valid": true,
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"v2": "5 errors in empty disjunction:\nconflicting values [] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
"v3": "conflicting values [] and {...} (mismatched types list and struct):\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (0 and 3):\n instance.json:1:1\n"
}
},
{
Expand All @@ -113,7 +114,8 @@
],
"valid": true,
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [1] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"v2": "5 errors in empty disjunction:\nconflicting values [1] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
"v3": "conflicting values [1] and {...} (mismatched types list and struct):\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (1 and 3):\n instance.json:1:1\n"
}
},
{
Expand All @@ -124,7 +126,8 @@
],
"valid": true,
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [1,2] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"v2": "5 errors in empty disjunction:\nconflicting values [1,2] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
"v3": "conflicting values [1,2] and {...} (mismatched types list and struct):\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (2 and 3):\n instance.json:1:1\n"
}
},
{
Expand Down Expand Up @@ -195,7 +198,8 @@
],
"valid": true,
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"v2": "5 errors in empty disjunction:\nconflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
"v3": "conflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (1 and 3):\n instance.json:1:1\n"
}
}
]
Expand Down Expand Up @@ -226,7 +230,8 @@
],
"valid": true,
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [1,null] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"v2": "5 errors in empty disjunction:\nconflicting values [1,null] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n",
"v3": "conflicting values [1,null] and {...} (mismatched types list and struct):\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\nincompatible list lengths (1 and 2):\n instance.json:1:1\n"
}
}
]
Expand Down
27 changes: 18 additions & 9 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/additionalProperties.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@
},
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
},
{
Expand Down Expand Up @@ -85,7 +86,8 @@
},
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
}
]
Expand Down Expand Up @@ -232,7 +234,8 @@
}
},
"skip": {
"v2": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n generated.cue:5:3\n"
"v2": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n generated.cue:5:3\n",
"v3": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n generated.cue:5:3\n"
},
"tests": [
{
Expand All @@ -242,7 +245,8 @@
},
"valid": true,
"skip": {
"v2": "could not compile schema"
"v2": "could not compile schema",
"v3": "could not compile schema"
}
},
{
Expand All @@ -253,7 +257,8 @@
},
"valid": false,
"skip": {
"v2": "could not compile schema"
"v2": "could not compile schema",
"v3": "could not compile schema"
}
}
]
Expand All @@ -276,7 +281,8 @@
"additionalProperties": false
},
"skip": {
"v2": "extract error: unsupported constraint \"dependentSchemas\""
"v2": "extract error: unsupported constraint \"dependentSchemas\"",
"v3": "extract error: unsupported constraint \"dependentSchemas\""
},
"tests": [
{
Expand All @@ -286,7 +292,8 @@
},
"valid": false,
"skip": {
"v2": "could not compile schema"
"v2": "could not compile schema",
"v3": "could not compile schema"
}
},
{
Expand All @@ -296,7 +303,8 @@
},
"valid": false,
"skip": {
"v2": "could not compile schema"
"v2": "could not compile schema",
"v3": "could not compile schema"
}
},
{
Expand All @@ -307,7 +315,8 @@
},
"valid": false,
"skip": {
"v2": "could not compile schema"
"v2": "could not compile schema",
"v3": "could not compile schema"
}
}
]
Expand Down
24 changes: 16 additions & 8 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/allOf.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@
},
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
},
{
Expand All @@ -52,7 +53,8 @@
},
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
},
{
Expand Down Expand Up @@ -118,7 +120,8 @@
},
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
},
{
Expand All @@ -129,7 +132,8 @@
},
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
},
{
Expand All @@ -140,7 +144,8 @@
},
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
},
{
Expand All @@ -150,7 +155,8 @@
},
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
}
]
Expand Down Expand Up @@ -213,7 +219,8 @@
"data": "foo",
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
}
]
Expand All @@ -233,7 +240,8 @@
"data": "foo",
"valid": false,
"skip": {
"v2": "unexpected success"
"v2": "unexpected success",
"v3": "unexpected success"
}
}
]
Expand Down
Loading

0 comments on commit e2685dd

Please sign in to comment.