From 646a37db8db27a6f6661f280c37962cd8a806a99 Mon Sep 17 00:00:00 2001 From: Charles Daniels Date: Fri, 23 Aug 2024 16:34:36 -0400 Subject: [PATCH] fix broken caps, add tests to embedded --- build/do.rq | 28 +++++++- internal/lsp/rego/builtins.go | 4 +- pkg/capabilities/capabilities.go | 6 +- pkg/capabilities/embedded/embedded.go | 68 +++++++++++++++++++ pkg/capabilities/embedded/embedded_test.go | 36 ++++++++++ pkg/capabilities/embedded/eopa/v0.100.5.json | 1 - pkg/capabilities/embedded/eopa/v0.100.6.json | 1 - pkg/capabilities/embedded/eopa/v0.100.7.json | 1 - pkg/capabilities/embedded/eopa/v0.49.0-5.json | 1 - pkg/capabilities/embedded/eopa/v0.49.0-6.json | 1 - pkg/capabilities/embedded/eopa/v0.49.0-7.json | 1 - pkg/capabilities/embedded/eopa/v0.49.0-8.json | 1 - pkg/capabilities/embedded/eopa/v1.15.0.json | 1 - pkg/capabilities/embedded/eopa/v1.4.1.json | 1 - pkg/capabilities/embedded/eopa/v1.5.0.json | 1 - 15 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 pkg/capabilities/embedded/embedded.go create mode 100644 pkg/capabilities/embedded/embedded_test.go delete mode 100644 pkg/capabilities/embedded/eopa/v0.100.5.json delete mode 100644 pkg/capabilities/embedded/eopa/v0.100.6.json delete mode 100644 pkg/capabilities/embedded/eopa/v0.100.7.json delete mode 100644 pkg/capabilities/embedded/eopa/v0.49.0-5.json delete mode 100644 pkg/capabilities/embedded/eopa/v0.49.0-6.json delete mode 100644 pkg/capabilities/embedded/eopa/v0.49.0-7.json delete mode 100644 pkg/capabilities/embedded/eopa/v0.49.0-8.json delete mode 100644 pkg/capabilities/embedded/eopa/v1.15.0.json delete mode 100644 pkg/capabilities/embedded/eopa/v1.4.1.json delete mode 100644 pkg/capabilities/embedded/eopa/v1.5.0.json diff --git a/build/do.rq b/build/do.rq index 87dfdfc8..b9b02f58 100755 --- a/build/do.rq +++ b/build/do.rq @@ -238,7 +238,25 @@ fetch_eopa_caps { { rq.error(sprintf("\nstdout: %s\nstderr: %s\n", [eopa_tags_result.stdout, eopa_tags_result.stderr])) | eopa_tags_result.exitcode != 0 } - eopa_tags := {r[2] | r := eopa_tags_result.stdout[_]} + + # We assume that tags and capabilities files are 1:1, but some EOPA + # release tags in the past did not correctly get capabilities files, so + # we eliminate them from consideration. + + known_bad_tags := { + "v0.100.5", # tag missing capabilities file + "v0.100.6", # tag missing capabilities file + "v0.100.7", # tag missing capabilities file + "v0.49.0-5", # tag missing capabilities file + "v0.49.0-6", # tag missing capabilities file + "v0.49.0-7", # tag missing capabilities file + "v0.49.0-8", # tag missing capabilities file + "v1.15.0", # tag missing capabilities file (misnamed v0.15.0) + "v1.4.1", # tag missing capabilities file + "v1.5.0", # tag missing capabilities file + } + + eopa_tags := {t | r := eopa_tags_result.stdout[_]; t := r[2] ; not known_bad_tags[t]} # Get a directory listing for the capabilities directory, filtering for # only nonzero size files with JSON extensions. The size check is to @@ -266,7 +284,7 @@ fetch_eopa_caps { r := rq.template("https://raw.githubusercontent.com/StyraInc/enterprise-opa/main/capabilities/{{.tag}}.json", {"tag": t}) } - print(sprintf("fetching %d capabilities files locally", [count(missing_locally)])) + print(sprintf("fetching %d capabilities files missing locally", [count(missing_locally)])) # Download the capabilities from the constructed URLs. new_caps := { @@ -275,7 +293,13 @@ fetch_eopa_caps { m := missing_locally[_] print(sprintf("\tfetcing %s", [m.remote])) resp := http.send({"url": m.remote, "method": "GET"}) + + { rq.error(sprintf("non-200 status code '%d' for URL '%s'", [resp.status_code, m.remote])) | resp.status_code != 200 } + c := resp.raw_body + + # sanity check in case we got the URL wrong + not regex.match("404: Not Found", c) } # Commit the retrieved content to disk. diff --git a/internal/lsp/rego/builtins.go b/internal/lsp/rego/builtins.go index dbd6a1a6..b0966805 100644 --- a/internal/lsp/rego/builtins.go +++ b/internal/lsp/rego/builtins.go @@ -7,8 +7,8 @@ import ( "github.com/open-policy-agent/opa/ast" ) -var builtInsLock = &sync.RWMutex{} // nolint:gochecknoglobals -var builtIns = builtinMap(ast.CapabilitiesForThisVersion()) //nolint:gochecknoglobals +var builtInsLock = &sync.RWMutex{} +var builtIns = builtinMap(ast.CapabilitiesForThisVersion()) // Update updates the builtins database with the provided capabilities. func UpdateBuiltins(caps *ast.Capabilities) { diff --git a/pkg/capabilities/capabilities.go b/pkg/capabilities/capabilities.go index 275e3003..36641664 100644 --- a/pkg/capabilities/capabilities.go +++ b/pkg/capabilities/capabilities.go @@ -16,7 +16,7 @@ import ( "github.com/coreos/go-semver/semver" - eopa_caps "github.com/styrainc/enterprise-opa/capabilities" + embedded "github.com/styrainc/regal/pkg/capabilities/embedded" ) const ( @@ -145,7 +145,7 @@ func lookupEmbeddedURL(parsedURL *url.URL) (*ast.Capabilities, error) { case engineOPA: return ast.LoadCapabilitiesVersion(version) case engineEOPA: - return eopa_caps.LoadCapabilitiesVersion(version) + return embedded.LoadCapabilitiesVersion(engineEOPA, version) default: return nil, fmt.Errorf("engine '%s' not present in embedded capabilities database", engine) } @@ -243,7 +243,7 @@ func List() (map[string][]string, error) { return nil, err } - eopaCaps, err := eopa_caps.LoadCapabilitiesVersions() + eopaCaps, err := embedded.LoadCapabilitiesVersions(engineEOPA) if err != nil { return nil, err } diff --git a/pkg/capabilities/embedded/embedded.go b/pkg/capabilities/embedded/embedded.go new file mode 100644 index 00000000..9c8d631e --- /dev/null +++ b/pkg/capabilities/embedded/embedded.go @@ -0,0 +1,68 @@ +// This file is copied and modified from: +// +// https://github.com/open-policy-agent/opa/blob/main/ast/capabilities.go +// +// It is made available under the Apache 2 license, which you can view here: +// +// https://github.com/open-policy-agent/opa/blob/main/LICENSE +// +// The original license disclaimer is included below: +// +// Copyright 2021 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. +// +// This file and the included helper methods allow Enterprise OPA's +// capabilities files to be consumed as a Go package. This mirrors the way Open +// Policy Agent does thing. + +// Package embedded handles embedding and access JSON files directly included in +// Regal from it's source repository +package embedded + +import ( + "bytes" + "embed" + "fmt" + "strings" + + "github.com/open-policy-agent/opa/ast" +) + +//go:embed */*.json +var FS embed.FS + +// LoadCapabilitiesVersion loads a JSON serialized capabilities structure from the specific version. +func LoadCapabilitiesVersion(engine, version string) (*ast.Capabilities, error) { + cvs, err := LoadCapabilitiesVersions(engine) + if err != nil { + return nil, err + } + + for _, cv := range cvs { + if cv == version { + cont, err := FS.ReadFile("eopa/" + cv + ".json") + if err != nil { + return nil, err + } + + return ast.LoadCapabilitiesJSON(bytes.NewReader(cont)) + } + + } + return nil, fmt.Errorf("(Regal embedded %s capabilities library) no capabilities version found %v", engine, version) +} + +// LoadCapabilitiesVersions loads all capabilities versions +func LoadCapabilitiesVersions(engine string) ([]string, error) { + ents, err := FS.ReadDir(engine) + if err != nil { + return nil, err + } + + capabilitiesVersions := make([]string, 0, len(ents)) + for _, ent := range ents { + capabilitiesVersions = append(capabilitiesVersions, strings.Replace(ent.Name(), ".json", "", 1)) + } + return capabilitiesVersions, nil +} diff --git a/pkg/capabilities/embedded/embedded_test.go b/pkg/capabilities/embedded/embedded_test.go new file mode 100644 index 00000000..f2e990fc --- /dev/null +++ b/pkg/capabilities/embedded/embedded_test.go @@ -0,0 +1,36 @@ +package embedded + +import "testing" + +func TestEmbeddedEOPA(t *testing.T) { + // As of 2024-08-23, there are 57 capabilities files in the EOPA repo. + // It follows that there should never be less than 54 valid + // capabilities in the embedded database. This is really just a sanity + // check to ensure the JSON files didn't get misplaced or something to + // that effect. + // + // This also ensures that all of the embedded capabilities files are + // valid JSON we can successfully marshal into *ast.Capabilities. + + versions, err := LoadCapabilitiesVersions("eopa") + if err != nil { + t.Fatal(err) + } + + if len(versions) < 54 { + t.Errorf("Expected at least 54 EOPA capabilities in the embedded database") + } + + for _, v := range versions { + caps, err := LoadCapabilitiesVersion("eopa", v) + + if err != nil { + t.Errorf("error with eopa capabilities version %s: %v", v, err) + } + + if len(caps.Builtins) < 1 { + t.Errorf("eopa capabilities version %s has no builtins", v) + } + } + +} diff --git a/pkg/capabilities/embedded/eopa/v0.100.5.json b/pkg/capabilities/embedded/eopa/v0.100.5.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v0.100.5.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v0.100.6.json b/pkg/capabilities/embedded/eopa/v0.100.6.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v0.100.6.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v0.100.7.json b/pkg/capabilities/embedded/eopa/v0.100.7.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v0.100.7.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v0.49.0-5.json b/pkg/capabilities/embedded/eopa/v0.49.0-5.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v0.49.0-5.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v0.49.0-6.json b/pkg/capabilities/embedded/eopa/v0.49.0-6.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v0.49.0-6.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v0.49.0-7.json b/pkg/capabilities/embedded/eopa/v0.49.0-7.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v0.49.0-7.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v0.49.0-8.json b/pkg/capabilities/embedded/eopa/v0.49.0-8.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v0.49.0-8.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v1.15.0.json b/pkg/capabilities/embedded/eopa/v1.15.0.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v1.15.0.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v1.4.1.json b/pkg/capabilities/embedded/eopa/v1.4.1.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v1.4.1.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file diff --git a/pkg/capabilities/embedded/eopa/v1.5.0.json b/pkg/capabilities/embedded/eopa/v1.5.0.json deleted file mode 100644 index 1becba2b..00000000 --- a/pkg/capabilities/embedded/eopa/v1.5.0.json +++ /dev/null @@ -1 +0,0 @@ -404: Not Found \ No newline at end of file