Skip to content

Commit

Permalink
encoding/jsonschema: add location information to test output
Browse files Browse the repository at this point in the history 
It's useful to know exactly where in the test data a test is,
so log that information. Also print a txtar file to make it
easier to reproduce test failures.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: Ibef118835a429437825eb443e265066737c5fcb1
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1200518
TryBot-Result: CUEcueckoo <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>
  • Loading branch information
@rogpeppe
rogpeppe committed Sep 3, 2024
1 parent c0fdf75 commit 1a2e26e
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 13 deletions.
74 changes: 65 additions & 9 deletions encoding/jsonschema/external_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package jsonschema_test

import (
stdjson "encoding/json"
"fmt"
"os"
"path"
Expand All @@ -28,6 +29,7 @@ import (
"cuelang.org/go/cue/cuecontext"
"cuelang.org/go/cue/errors"
"cuelang.org/go/cue/format"
"cuelang.org/go/cue/token"
"cuelang.org/go/encoding/json"
"cuelang.org/go/encoding/jsonschema"
"cuelang.org/go/encoding/jsonschema/internal/externaltest"
Expand Down Expand Up @@ -100,23 +102,32 @@ func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schem
qt.Assert(t, qt.IsNil(err))
schemaValue = ctx.CompileBytes(b, cue.Filename("generated.cue"))
if err := schemaValue.Err(); err != nil {
t.Logf("extracted schema: %q", b)
extractErr = fmt.Errorf("cannot compile resulting schema: %v", errors.Details(err, nil))
}
}

if extractErr != nil {
t.Logf("location: %v", testdataPos(s))
t.Logf("txtar:\n%s", schemaFailureTxtar(s))
for _, test := range s.Tests {
t.Run("", func(t *testing.T) {
testFailed(t, &test.Skip, "could not compile schema")
testFailed(t, &test.Skip, test, "could not compile schema")
})
}
testFailed(t, &s.Skip, fmt.Sprintf("extract error: %v", extractErr))
testFailed(t, &s.Skip, s, fmt.Sprintf("extract error: %v", extractErr))
return
}
testSucceeded(t, &s.Skip)
testSucceeded(t, &s.Skip, s)

for _, test := range s.Tests {
t.Run(testName(test.Description), func(t *testing.T) {
defer func() {
if t.Failed() || testing.Verbose() {
t.Logf("txtar:\n%s", testCaseTxtar(s, test))
}
}()
t.Logf("location: %v", testdataPos(test))
instAST, err := json.Extract("instance.json", test.Data)
if err != nil {
t.Fatal(err)
Expand All @@ -129,21 +140,60 @@ func runExternalSchemaTests(t *testing.T, filename string, s *externaltest.Schem
err = instValue.Unify(schemaValue).Err()
if test.Valid {
if err != nil {
testFailed(t, &test.Skip, errors.Details(err, nil))
testFailed(t, &test.Skip, test, errors.Details(err, nil))
} else {
testSucceeded(t, &test.Skip)
testSucceeded(t, &test.Skip, test)
}
} else {
if err == nil {
testFailed(t, &test.Skip, "unexpected success")
testFailed(t, &test.Skip, test, "unexpected success")
} else {
testSucceeded(t, &test.Skip)
testSucceeded(t, &test.Skip, test)
}
}
})
}
}

// testCaseTxtar returns a testscript that runs the given test.
func testCaseTxtar(s *externaltest.Schema, test *externaltest.Test) string {
var buf strings.Builder
fmt.Fprintf(&buf, "env CUE_EXPERIMENT=evalv3\n")
fmt.Fprintf(&buf, "exec cue def json+jsonschema: schema.json\n")
if !test.Valid {
buf.WriteString("! ")
}
// TODO add $schema when one isn't already present?
fmt.Fprintf(&buf, "exec cue vet -c instance.json json+jsonschema: schema.json\n")
fmt.Fprintf(&buf, "\n")
fmt.Fprintf(&buf, "-- schema.json --\n%s\n", indentJSON(s.Schema))
fmt.Fprintf(&buf, "-- instance.json --\n%s\n", indentJSON(test.Data))
return buf.String()
}

// testCaseTxtar returns a testscript that decodes the given schema.
func schemaFailureTxtar(s *externaltest.Schema) string {
var buf strings.Builder
fmt.Fprintf(&buf, "env CUE_EXPERIMENT=evalv3\n")
fmt.Fprintf(&buf, "exec cue def -o schema.cue json+jsonschema: schema.json\n")
fmt.Fprintf(&buf, "exec cat schema.cue\n")
fmt.Fprintf(&buf, "exec cue vet schema.cue\n")
fmt.Fprintf(&buf, "-- schema.json --\n%s\n", indentJSON(s.Schema))
return buf.String()
}

func indentJSON(x stdjson.RawMessage) []byte {
data, err := stdjson.MarshalIndent(x, "", "\t")
if err != nil {
panic(err)
}
return data
}

type positioner interface {
Pos() token.Pos
}

// testName returns a test name that doesn't contain any
// slashes because slashes muck with matching.
func testName(s string) string {
Expand All @@ -153,7 +203,7 @@ func testName(s string) string {
// testFailed marks the current test as failed with the
// given error message, and updates the
// skip field pointed to by skipField if necessary.
func testFailed(t *testing.T, skipField *string, errStr string) {
func testFailed(t *testing.T, skipField *string, p positioner, errStr string) {
if cuetest.UpdateGoldenFiles {
if *skipField == "" && !allowRegressions() {
t.Fatalf("test regression; was succeeding, now failing: %v", errStr)
Expand All @@ -169,7 +219,7 @@ func testFailed(t *testing.T, skipField *string, errStr string) {

// testFails marks the current test as succeeded and updates the
// skip field pointed to by skipField if necessary.
func testSucceeded(t *testing.T, skipField *string) {
func testSucceeded(t *testing.T, skipField *string, p positioner) {
if cuetest.UpdateGoldenFiles {
*skipField = ""
return
Expand All @@ -179,6 +229,12 @@ func testSucceeded(t *testing.T, skipField *string) {
}
}

func testdataPos(p positioner) token.Position {
pp := p.Pos().Position()
pp.Filename = path.Join(testDir, pp.Filename)
return pp
}

func allowRegressions() bool {
return os.Getenv("CUE_ALLOW_REGRESSIONS") != ""
}
Expand Down
28 changes: 24 additions & 4 deletions encoding/jsonschema/internal/externaltest/tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ import (
"cuelang.org/go/cue/cuecontext"
"cuelang.org/go/cue/interpreter/embed"
"cuelang.org/go/cue/load"
"cuelang.org/go/cue/token"
)

type Schema struct {
location
Description string `json:"description"`
Comment string `json:"comment,omitempty"`
Schema stdjson.RawMessage `json:"schema"`
Expand All @@ -23,13 +25,23 @@ type Schema struct {
}

type Test struct {
location
Description string `json:"description"`
Comment string `json:"comment,omitempty"`
Data stdjson.RawMessage `json:"data"`
Valid bool `json:"valid"`
Skip string `json:"skip,omitempty"`
}

type location struct {
root cue.Value
path cue.Path
}

func (loc location) Pos() token.Pos {
return loc.root.LookupPath(loc.path).Pos()
}

func ParseTestData(data []byte) ([]*Schema, error) {
var schemas []*Schema
if err := json.Unmarshal(data, &schemas); err != nil {
Expand Down Expand Up @@ -81,7 +93,7 @@ func ReadTestDir(dir string) (tests map[string][]*Schema, err error) {
inst := load.Instances([]string{"."}, &load.Config{
Dir: dir,
})[0]
if err != nil {
if err := inst.Err; err != nil {
return nil, err
}
ctx := cuecontext.New(cuecontext.Interpreter(embed.New()))
Expand All @@ -97,9 +109,17 @@ func ReadTestDir(dir string) (tests map[string][]*Schema, err error) {
return nil, err
}
// Fix up the raw JSON data to avoid running into some decode issues.
for _, schemas := range tests {
for _, schema := range schemas {
for _, test := range schema.Tests {
for filename, schemas := range tests {
for i, schema := range schemas {
schema.location = location{
root: val,
path: cue.MakePath(cue.Str(filename), cue.Index(i)),
}
for j, test := range schema.Tests {
test.location = location{
root: val,
path: cue.MakePath(cue.Str(filename), cue.Index(i), cue.Str("tests"), cue.Index(j)),
}
if len(test.Data) == 0 {
// See https://github.com/cue-lang/cue/issues/3397
test.Data = []byte("null")
Expand Down

0 comments on commit 1a2e26e

Please sign in to comment.