Skip to content

Commit

Permalink
encoding/jsonschema: prepare for matrix test runs
Browse files Browse the repository at this point in the history 
Currently we are just running the external tests on the
old evaluator, but we want confidence in what's happening
in both evaluators.

In order to do so, we will record skip reasons for both
old and new evaluators, so prepare for that by making
the `skip` field into a struct and moving all the
existing skip fields into a field inside that, corresponding
to the point in the matrix that is being run currently.

In the next CL, we will actually run the matrix tests
and populate the `skip.v3` field.

Most of this CL has been accomplished with an automated script.

Also, in passing, remove the `externaltest.ParseTestData`
function as it's not used.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: I6f59c6eb837a7c15eb7467f6860167534b6d1e75
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200519
TryBot-Result: CUEcueckoo <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>
  • Loading branch information
@rogpeppe
rogpeppe committed Sep 3, 2024
1 parent 1a2e26e commit 9959a3e
Show file tree
Hide file tree
Showing 261 changed files with 14,494 additions and 4,852 deletions.
14 changes: 7 additions & 7 deletions encoding/jsonschema/external_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,28 +203,28 @@ func testName(s string) string {
// testFailed marks the current test as failed with the
// given error message, and updates the
// skip field pointed to by skipField if necessary.
func testFailed(t *testing.T, skipField *string, p positioner, errStr string) {
func testFailed(t *testing.T, skipField *externaltest.Skip, p positioner, errStr string) {
if cuetest.UpdateGoldenFiles {
if *skipField == "" && !allowRegressions() {
if *skipField == nil && !allowRegressions() {
t.Fatalf("test regression; was succeeding, now failing: %v", errStr)
}
*skipField = errStr
*skipField = externaltest.Skip{"v2": errStr}
return
}
if *skipField != "" {
if *skipField != nil {
t.Skipf("skipping due to known error: %v", *skipField)
}
t.Fatal(errStr)
}

// testFails marks the current test as succeeded and updates the
// skip field pointed to by skipField if necessary.
func testSucceeded(t *testing.T, skipField *string, p positioner) {
func testSucceeded(t *testing.T, skipField *externaltest.Skip, p positioner) {
if cuetest.UpdateGoldenFiles {
*skipField = ""
*skipField = nil
return
}
if *skipField != "" {
if *skipField != nil {
t.Fatalf("unexpectedly more correct behavior (test success) on skipped test")
}
}
Expand Down
21 changes: 10 additions & 11 deletions encoding/jsonschema/internal/externaltest/tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package externaltest

import (
"bytes"
"encoding/json"
stdjson "encoding/json"
"fmt"
"os"
Expand All @@ -20,7 +19,7 @@ type Schema struct {
Description string `json:"description"`
Comment string `json:"comment,omitempty"`
Schema stdjson.RawMessage `json:"schema"`
Skip string `json:"skip,omitempty"`
Skip Skip `json:"skip,omitempty"`
Tests []*Test `json:"tests"`
}

Expand All @@ -30,9 +29,17 @@ type Test struct {
Comment string `json:"comment,omitempty"`
Data stdjson.RawMessage `json:"data"`
Valid bool `json:"valid"`
Skip string `json:"skip,omitempty"`
Skip Skip `json:"skip,omitempty"`
}

// Skip records information about whether a given schema
// or test will be skipped when testing. If not present,
// the test will be expected to pass.
//
// Each key in the map represents the name of a point
// in the cuetdtest matrix.
type Skip map[string]string

type location struct {
root cue.Value
path cue.Path
Expand All @@ -42,14 +49,6 @@ func (loc location) Pos() token.Pos {
return loc.root.LookupPath(loc.path).Pos()
}

func ParseTestData(data []byte) ([]*Schema, error) {
var schemas []*Schema
if err := json.Unmarshal(data, &schemas); err != nil {
return nil, err
}
return schemas, nil
}

// WriteTestDir writes test data files as read by ReadTestDir
// to the given directory. The keys of tests are filenames relative
// to dir.
Expand Down
17 changes: 13 additions & 4 deletions encoding/jsonschema/testdata/external/config.cue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@ allTests: [_]: [... #Schema]
// skip is not part of the orginal test data, but
// is inserted by our test logic (when CUE_UPDATE=1)
// to indicate which tests are passing and which
// are failing. The text indicates some reason as to
// why the schema is skipped.
skip?: string
// are failing. See #Skip for details.
skip?: #Skip
}

#Test: {
Expand All @@ -35,5 +34,15 @@ allTests: [_]: [... #Schema]
// to indicate which tests are passing and which
// are failing. The text indicates some reason as to
// why the test is skipped.
skip?: string
skip?: #Skip
}

// #Skip indicates that a test or schema should be skipped
// in the external test. The text indicates some reason as to
// why the schema is skipped for a given evaluator version.
//
// If all fields are empty, the skip field itself should be omitted.
#Skip: {
v2?: string
v3?: string
}
20 changes: 15 additions & 5 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/additionalItems.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,19 @@
"description": "empty array",
"data": [],
"valid": true,
"skip": "5 errors in empty disjunction:\nconflicting values [] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
}
},
{
"description": "fewer number of items present (1)",
"data": [
1
],
"valid": true,
"skip": "5 errors in empty disjunction:\nconflicting values [1] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [1] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
}
},
{
"description": "fewer number of items present (2)",
Expand All @@ -119,7 +123,9 @@
2
],
"valid": true,
"skip": "5 errors in empty disjunction:\nconflicting values [1,2] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [1,2] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:45\n instance.json:1:1\nconflicting values bool and [1,2] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,2] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,2] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,2] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
}
},
{
"description": "equal number of items present",
Expand Down Expand Up @@ -188,7 +194,9 @@
false
],
"valid": true,
"skip": "5 errors in empty disjunction:\nconflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [1,\"foo\",false] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,\"foo\",false] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,\"foo\",false] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,\"foo\",false] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,\"foo\",false] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
}
}
]
},
Expand Down Expand Up @@ -217,7 +225,9 @@
null
],
"valid": true,
"skip": "5 errors in empty disjunction:\nconflicting values [1,null] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
"skip": {
"v2": "5 errors in empty disjunction:\nconflicting values [1,null] and {...} (mismatched types list and struct):\n generated.cue:2:1\n generated.cue:2:41\n instance.json:1:1\nconflicting values bool and [1,null] (mismatched types bool and list):\n generated.cue:2:8\n instance.json:1:1\nconflicting values null and [1,null] (mismatched types null and list):\n generated.cue:2:1\n instance.json:1:1\nconflicting values number and [1,null] (mismatched types number and list):\n generated.cue:2:15\n instance.json:1:1\nconflicting values string and [1,null] (mismatched types string and list):\n generated.cue:2:24\n instance.json:1:1\n"
}
}
]
},
Expand Down
36 changes: 27 additions & 9 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/additionalProperties.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@
"quux": "boom"
},
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
},
{
"description": "ignores arrays",
Expand Down Expand Up @@ -82,7 +84,9 @@
"élmény": 2
},
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
}
]
},
Expand Down Expand Up @@ -227,15 +231,19 @@
"type": "number"
}
},
"skip": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n generated.cue:5:3\n",
"skip": {
"v2": "extract error: cannot compile resulting schema: reference \"strings\" in label expression refers to field against which it would be matched:\n generated.cue:5:3\n"
},
"tests": [
{
"description": "Valid against both keywords",
"data": {
"apple": 4
},
"valid": true,
"skip": "could not compile schema"
"skip": {
"v2": "could not compile schema"
}
},
{
"description": "Valid against propertyNames, but not additionalProperties",
Expand All @@ -244,7 +252,9 @@
"pear": "available"
},
"valid": false,
"skip": "could not compile schema"
"skip": {
"v2": "could not compile schema"
}
}
]
},
Expand All @@ -265,23 +275,29 @@
},
"additionalProperties": false
},
"skip": "extract error: unsupported constraint \"dependentSchemas\"",
"skip": {
"v2": "extract error: unsupported constraint \"dependentSchemas\""
},
"tests": [
{
"description": "additionalProperties doesn't consider dependentSchemas",
"data": {
"foo": ""
},
"valid": false,
"skip": "could not compile schema"
"skip": {
"v2": "could not compile schema"
}
},
{
"description": "additionalProperties can't see bar",
"data": {
"bar": ""
},
"valid": false,
"skip": "could not compile schema"
"skip": {
"v2": "could not compile schema"
}
},
{
"description": "additionalProperties can't see bar even when foo2 is present",
Expand All @@ -290,7 +306,9 @@
"bar": ""
},
"valid": false,
"skip": "could not compile schema"
"skip": {
"v2": "could not compile schema"
}
}
]
}
Expand Down
32 changes: 24 additions & 8 deletions encoding/jsonschema/testdata/external/tests/draft2019-09/allOf.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,19 @@
"foo": "baz"
},
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
},
{
"description": "mismatch first",
"data": {
"bar": 2
},
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
},
{
"description": "wrong type",
Expand Down Expand Up @@ -113,7 +117,9 @@
"baz": null
},
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
},
{
"description": "mismatch first allOf",
Expand All @@ -122,7 +128,9 @@
"baz": null
},
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
},
{
"description": "mismatch second allOf",
Expand All @@ -131,15 +139,19 @@
"bar": 2
},
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
},
{
"description": "mismatch both",
"data": {
"bar": 2
},
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
}
]
},
Expand Down Expand Up @@ -200,7 +212,9 @@
"description": "any value is invalid",
"data": "foo",
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
}
]
},
Expand All @@ -218,7 +232,9 @@
"description": "any value is invalid",
"data": "foo",
"valid": false,
"skip": "unexpected success"
"skip": {
"v2": "unexpected success"
}
}
]
},
Expand Down
Loading

0 comments on commit 9959a3e

Please sign in to comment.