From 798cc136d72255c1b8e8e0b4aa2a27a6b806b54a Mon Sep 17 00:00:00 2001 From: Teodora Sandu Date: Mon, 13 Dec 2021 17:39:58 +0000 Subject: [PATCH] feat: include metrics about number of custom rules --- internal/build.go | 21 +++++++++- internal/build_test.go | 55 ++++++++++++++++++++++++++ util/inspect.go | 65 +++++++++++++++++++++++++++++++ util/inspect_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 util/inspect.go create mode 100644 util/inspect_test.go diff --git a/internal/build.go b/internal/build.go index f6e1add..282d186 100644 --- a/internal/build.go +++ b/internal/build.go @@ -3,8 +3,10 @@ package internal import ( "bytes" "context" + "fmt" "io" "os" + "path" "strings" "github.com/open-policy-agent/opa/ast" @@ -30,6 +32,19 @@ type BuildCommandParams struct { func RunBuild(args []string, params *BuildCommandParams) error { buf := bytes.NewBuffer(nil) + var metadataFile = "" + rules, err := util.RetrieveRules(args) + if err == nil { + // choose either one of the locations for the metadata.json + // it will be included in the OPA generated data.json + metadataFile = path.Join(args[0], "metadata.json") + err = os.WriteFile(metadataFile, []byte(fmt.Sprintf("{\"numberOfRules\": %d}", len(rules))), + 0644) + if err != nil { + return err + } + } + var capabilities *ast.Capabilities // if capabilities are not provided as a cmd flag, // then ast.CapabilitiesForThisVersion must be called @@ -49,7 +64,7 @@ func RunBuild(args []string, params *BuildCommandParams) error { WithPaths(args...). WithFilter(buildCommandLoaderFilter(false, params.Ignore)) - err := compiler.Build(context.Background()) + err = compiler.Build(context.Background()) if err != nil { return err } @@ -64,6 +79,10 @@ func RunBuild(args []string, params *BuildCommandParams) error { return err } + if metadataFile != "" { + _ = os.Remove(metadataFile) + } + return out.Close() } diff --git a/internal/build_test.go b/internal/build_test.go index 066f145..875edf8 100644 --- a/internal/build_test.go +++ b/internal/build_test.go @@ -2,6 +2,7 @@ package internal import ( "archive/tar" + "bytes" "compress/gzip" "io" "os" @@ -268,3 +269,57 @@ func TestBuildRespectsCapabilitiesFailure(t *testing.T) { assert.Contains(t, err.Error(), "undefined function is_foo") }) } + +func TestBuildProducesMetadata(t *testing.T) { + files := map[string]string{ + "test.rego": ` + package test + msg = { + "publicId": + "1" + } + `, + } + + test.WithTempFS(files, func(root string) { + buildParams := mockBuildParams() + buildParams.OutputFile = path.Join(root, "bundle.tar.gz") + err := buildParams.Target.Set(TargetRego) + assert.Nil(t, err) + + err = RunBuild([]string{root}, buildParams) + assert.Nil(t, err) + + _, err = loader.NewFileLoader().AsBundle(buildParams.OutputFile) + assert.Nil(t, err) + + // Check that manifest is not written given no input manifest and no other flags + f, err := os.Open(buildParams.OutputFile) + assert.Nil(t, err) + defer f.Close() + + gr, err := gzip.NewReader(f) + assert.Nil(t, err) + + tr := tar.NewReader(gr) + + for { + f, err := tr.Next() + if err == io.EOF { + break + } + assert.Nil(t, err) + + if f.Name == "/data.json" { + data := new(bytes.Buffer) + _, err := data.ReadFrom(tr) + assert.Nil(t, err) + assert.Contains(t, data.String(), "{\"numberOfRules\":1}") + } + + if f.Name == "/metadata.json" { + t.Fatal("unexpected file:", f.Name) + } + } + }) +} diff --git a/util/inspect.go b/util/inspect.go new file mode 100644 index 0000000..61acb6c --- /dev/null +++ b/util/inspect.go @@ -0,0 +1,65 @@ +package util + +import ( + "github.com/open-policy-agent/opa/loader" + "os" + "path/filepath" + "regexp" + "strings" +) + +func RetrieveRules(paths []string) ([]string, error) { + fileNames, err := findRegoFiles(paths) + if err != nil { + return []string{}, err + } + + var publicIds = []string{} + for _, fileName := range fileNames { + publicId, err := getPublicIdFromFile(fileName) + if err != nil { + return []string{}, err + } + if publicId != "" { + publicIds = append(publicIds, publicId) + } + } + return publicIds, nil +} + +func findRegoFiles(paths []string) ([]string, error) { + fileNames := []string{} + + result, err := loader.AllRegos(paths) + if err != nil { + return fileNames, err + } + + for _, module := range result.Modules { + fileName := filepath.Clean(module.Name) + if !strings.Contains(fileName, "_test.rego") { + fileNames = append(fileNames, fileName) + } + } + + return fileNames, nil +} + +func getPublicIdFromFile(fileName string) (string, error) { + data, err := os.ReadFile(fileName) + if err != nil { + return "", err + } + + publicId := extractPublicIdFromRego(string(data)) + return publicId, nil +} + +func extractPublicIdFromRego(rego string) string { + re := regexp.MustCompile("\"publicId\"\\s*:\\s*\"(.*?)\"") + match := re.FindStringSubmatch(rego) + if len(match) > 0 { + return match[1] + } + return "" +} diff --git a/util/inspect_test.go b/util/inspect_test.go new file mode 100644 index 0000000..d2464ba --- /dev/null +++ b/util/inspect_test.go @@ -0,0 +1,88 @@ +package util + +import ( + "github.com/open-policy-agent/opa/util/test" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestRetrieveRulesWitInvalidPath(t *testing.T) { + _, err := RetrieveRules([]string{"./invalid"}) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "./invalid: no such file or directory") +} + +func TestRetrieveRulesWithNoFiles(t *testing.T) { + files := map[string]string{} + test.WithTempFS(files, func(root string) { + rules, err := RetrieveRules([]string{root}) + assert.Nil(t, err) + assert.Equal(t, 0, len(rules)) + }) +} + +func TestRetrieveRulesWithNoRules(t *testing.T) { + files := map[string]string{ + "test.json": ` + test + `, + } + + test.WithTempFS(files, func(root string) { + rules, err := RetrieveRules([]string{root}) + assert.Nil(t, err) + assert.Equal(t, 0, len(rules)) + }) +} + +func TestRetrieveRulesWithRulesWithoutPublicId(t *testing.T) { + files := map[string]string{ + "test.rego": ` + package test + msg = { + "publicI": + "1" + } + `, + } + + test.WithTempFS(files, func(root string) { + rules, err := RetrieveRules([]string{root}) + assert.Nil(t, err) + assert.Equal(t, 0, len(rules)) + }) +} + +func TestRetrieveRulesWithRulesWithPublicId(t *testing.T) { + files := map[string]string{ + "test1.rego": ` + package test + msg = { + "publicId": + "1" + } + `, + "test2.rego": ` + package test + msg = { + "publicId": + "2" + } + `, + "test2_test.rego": ` + package test + msg = { + "publicId": + "3" + } + `, + } + + test.WithTempFS(files, func(root string) { + rules, err := RetrieveRules([]string{root}) + assert.Nil(t, err) + assert.Equal(t, 2, len(rules)) + assert.Equal(t, "1", rules[0]) + assert.Equal(t, "2", rules[1]) + }) +}