fix broken caps, add tests to embedded

build/do.rq

@@ -238,7 +238,25 @@ fetch_eopa_caps {
 
 	{ rq.error(sprintf("\nstdout: %s\nstderr: %s\n", [eopa_tags_result.stdout, eopa_tags_result.stderr])) | eopa_tags_result.exitcode != 0 }
 
-	eopa_tags := {r[2] | r := eopa_tags_result.stdout[_]}
+
+	# We assume that tags and capabilities files are 1:1, but some EOPA
+	# release tags in the past did not correctly get capabilities files, so
+	# we eliminate them from consideration.
+
+	known_bad_tags := {
+		"v0.100.5",  # tag missing capabilities file
+		"v0.100.6",  # tag missing capabilities file
+		"v0.100.7",  # tag missing capabilities file
+		"v0.49.0-5", # tag missing capabilities file
+		"v0.49.0-6", # tag missing capabilities file
+		"v0.49.0-7", # tag missing capabilities file
+		"v0.49.0-8", # tag missing capabilities file
+		"v1.15.0",   # tag missing capabilities file (misnamed v0.15.0)
+		"v1.4.1",    # tag missing capabilities file
+		"v1.5.0",    # tag missing capabilities file
+	}
+
+	eopa_tags := {t | r := eopa_tags_result.stdout[_]; t := r[2] ; not known_bad_tags[t]}
 
 	# Get a directory listing for the capabilities directory, filtering for
 	# only nonzero size files with JSON extensions. The size check is to
@@ -266,7 +284,7 @@ fetch_eopa_caps {
 		r := rq.template("https://raw.githubusercontent.com/StyraInc/enterprise-opa/main/capabilities/{{.tag}}.json", {"tag": t})
 	}
 
-	print(sprintf("fetching %d capabilities files locally", [count(missing_locally)]))
+	print(sprintf("fetching %d capabilities files missing locally", [count(missing_locally)]))
 
 	# Download the capabilities from the constructed URLs.
 	new_caps := {
@@ -275,7 +293,13 @@ fetch_eopa_caps {
 		m := missing_locally[_]
 		print(sprintf("\tfetcing %s", [m.remote]))
 		resp := http.send({"url": m.remote, "method": "GET"})
+
+		{ rq.error(sprintf("non-200 status code '%d' for URL '%s'", [resp.status_code, m.remote])) | resp.status_code != 200 }
+
 		c := resp.raw_body
+
+		# sanity check in case we got the URL wrong
+		not regex.match("404: Not Found", c)
 	}
 
 	# Commit the retrieved content to disk.

internal/lsp/rego/builtins.go

@@ -7,8 +7,8 @@ import (
 	"github.com/open-policy-agent/opa/ast"
 )
 
-var builtInsLock = &sync.RWMutex{}                          // nolint:gochecknoglobals
-var builtIns = builtinMap(ast.CapabilitiesForThisVersion()) //nolint:gochecknoglobals
+var builtInsLock = &sync.RWMutex{}
+var builtIns = builtinMap(ast.CapabilitiesForThisVersion())
 
 // Update updates the builtins database with the provided capabilities.
 func UpdateBuiltins(caps *ast.Capabilities) {

pkg/capabilities/capabilities.go

@@ -16,7 +16,7 @@ import (
 
 	"github.com/coreos/go-semver/semver"
 
-	eopa_caps "github.com/styrainc/enterprise-opa/capabilities"
+	embedded "github.com/styrainc/regal/pkg/capabilities/embedded"
 )
 
 const (
@@ -145,7 +145,7 @@ func lookupEmbeddedURL(parsedURL *url.URL) (*ast.Capabilities, error) {
 	case engineOPA:
 		return ast.LoadCapabilitiesVersion(version)
 	case engineEOPA:
-		return eopa_caps.LoadCapabilitiesVersion(version)
+		return embedded.LoadCapabilitiesVersion(engineEOPA, version)
 	default:
 		return nil, fmt.Errorf("engine '%s' not present in embedded capabilities database", engine)
 	}
@@ -243,7 +243,7 @@ func List() (map[string][]string, error) {
 		return nil, err
 	}
 
-	eopaCaps, err := eopa_caps.LoadCapabilitiesVersions()
+	eopaCaps, err := embedded.LoadCapabilitiesVersions(engineEOPA)
 	if err != nil {
 		return nil, err
 	}

pkg/capabilities/embedded/embedded.go

@@ -0,0 +1,68 @@
+// This file is copied and modified from:
+//
+// https://github.com/open-policy-agent/opa/blob/main/ast/capabilities.go
+//
+// It is made available under the Apache 2 license, which you can view here:
+//
+// https://github.com/open-policy-agent/opa/blob/main/LICENSE
+//
+// The original license disclaimer is included below:
+//
+// Copyright 2021 The OPA Authors.  All rights reserved.
+// Use of this source code is governed by an Apache2
+// license that can be found in the LICENSE file.
+//
+// This file and the included helper methods allow Enterprise OPA's
+// capabilities files to be consumed as a Go package. This mirrors the way Open
+// Policy Agent does thing.
+
+// Package embedded handles embedding and access JSON files directly included in
+// Regal from it's source repository
+package embedded
+
+import (
+	"bytes"
+	"embed"
+	"fmt"
+	"strings"
+
+	"github.com/open-policy-agent/opa/ast"
+)
+
+//go:embed */*.json
+var FS embed.FS
+
+// LoadCapabilitiesVersion loads a JSON serialized capabilities structure from the specific version.
+func LoadCapabilitiesVersion(engine, version string) (*ast.Capabilities, error) {
+	cvs, err := LoadCapabilitiesVersions(engine)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, cv := range cvs {
+		if cv == version {
+			cont, err := FS.ReadFile("eopa/" + cv + ".json")
+			if err != nil {
+				return nil, err
+			}
+
+			return ast.LoadCapabilitiesJSON(bytes.NewReader(cont))
+		}
+
+	}
+	return nil, fmt.Errorf("(Regal embedded %s capabilities library) no capabilities version found %v", engine, version)
+}
+
+// LoadCapabilitiesVersions loads all capabilities versions
+func LoadCapabilitiesVersions(engine string) ([]string, error) {
+	ents, err := FS.ReadDir(engine)
+	if err != nil {
+		return nil, err
+	}
+
+	capabilitiesVersions := make([]string, 0, len(ents))
+	for _, ent := range ents {
+		capabilitiesVersions = append(capabilitiesVersions, strings.Replace(ent.Name(), ".json", "", 1))
+	}
+	return capabilitiesVersions, nil
+}

pkg/capabilities/embedded/embedded_test.go

@@ -0,0 +1,36 @@
+package embedded
+
+import "testing"
+
+func TestEmbeddedEOPA(t *testing.T) {
+	// As of 2024-08-23, there are 57 capabilities files in the EOPA repo.
+	// It follows that there should never be less than 54 valid
+	// capabilities in the embedded database. This is really just a sanity
+	// check to ensure the JSON files didn't get misplaced or something to
+	// that effect.
+	//
+	// This also ensures that all of the embedded capabilities files are
+	// valid JSON we can successfully marshal into *ast.Capabilities.
+
+	versions, err := LoadCapabilitiesVersions("eopa")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(versions) < 54 {
+		t.Errorf("Expected at least 54 EOPA capabilities in the embedded database")
+	}
+
+	for _, v := range versions {
+		caps, err := LoadCapabilitiesVersion("eopa", v)
+
+		if err != nil {
+			t.Errorf("error with eopa capabilities version %s: %v", v, err)
+		}
+
+		if len(caps.Builtins) < 1 {
+			t.Errorf("eopa capabilities version %s has no builtins", v)
+		}
+	}
+
+}