fix: playbook env templating

cmd/catalog.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -24,56 +25,52 @@ var Catalog = &cobra.Command{
 }
 
 func parseQuery(args []string) query.SearchResourcesRequest {
-	request := query.SearchResourcesRequest{}
+	logger.Infof("Search query %v", args)
+	request := query.SearchResourcesRequest{
+		Limit: 5,
+	}
 	tags := make(map[string]string)
-	var configTypes []string
-	for _, arg := range args[1:] {
+	selector := types.ResourceSelector{
+		Cache: "no-cache",
+	}
+	for _, arg := range args {
 		parts := strings.Split(arg, "=")
 		if len(parts) != 2 {
 			logger.Warnf("Invalid param: %s", arg)
 			continue
 		}
 
 		switch parts[0] {
-		case "config":
-			request.Configs = append(request.Configs, types.ResourceSelector{Search: parts[1]})
-		case "config_id":
-			request.Configs = append(request.Configs, types.ResourceSelector{ID: parts[1]})
-		case "component":
-			request.Components = append(request.Components, types.ResourceSelector{Search: parts[1]})
-		case "component_id":
-			request.Components = append(request.Components, types.ResourceSelector{ID: parts[1]})
-		case "check":
-			request.Checks = append(request.Checks, types.ResourceSelector{Search: parts[1]})
-		case "check_id":
-			request.Checks = append(request.Checks, types.ResourceSelector{ID: parts[1]})
+		case "limit":
+			l, _ := strconv.Atoi(parts[1])
+			request.Limit = l
+		case "search":
+			selector.Search = parts[1]
+		case "scope":
+			selector.Scope = parts[1]
 		case "type":
-			configTypes = append(configTypes, parts[1])
+			selector.Types = append(selector.Types, parts[1])
+		case "name":
+			selector.Name = parts[1]
+		case "namespace":
+			selector.Namespace = parts[1]
+		case "id":
+			selector.ID = parts[1]
+		case "status":
+			selector.Statuses = append(selector.Statuses, parts[1])
 		default:
 			tags[parts[0]] = parts[1]
 		}
-
-	}
-	if len(configTypes) > 0 {
-		for i := range request.Configs {
-			request.Configs[i].Types = configTypes
-		}
-		for i := range request.Components {
-			request.Components[i].Types = configTypes
-		}
 	}
-	if len(tags) > 0 {
-		for i := range request.Configs {
-			for k, v := range tags {
-				request.Configs[i].LabelSelector += fmt.Sprintf(" %s=%s", k, v)
-			}
-		}
-		for i := range request.Components {
-			for k, v := range tags {
-				request.Components[i].LabelSelector += fmt.Sprintf(" %s=%s", k, v)
-			}
+
+	for k, v := range tags {
+		if strings.HasPrefix(k, "@") {
+			selector.TagSelector += fmt.Sprintf(" %s=%s", k[1:], v)
+		} else {
+			selector.LabelSelector += fmt.Sprintf(" %s=%s", k, v)
 		}
 	}
+	request.Configs = []types.ResourceSelector{selector}
 
 	return request
 }
@@ -120,7 +117,7 @@ var Query = &cobra.Command{
 				os.Exit(1)
 			}
 
-			logger.Infof("Waiting %s for resources to be discovered...", catalogWaitFor-time.Since(start))
+			logger.Infof("Waiting %s for %s", req, catalogWaitFor-time.Since(start))
 			time.Sleep(3 * time.Second)
 
 		}
@@ -134,7 +131,7 @@ var Query = &cobra.Command{
 
 func init() {
 	Query.Flags().StringVarP(&catalogOutfile, "out-file", "o", "", "Write catalog output to a file instead of stdout")
-	Query.Flags().StringVarP(&catalogOutformat, "out-format", "f", "yaml", "Format of output file or stdout (yaml or json)")
+	Query.Flags().StringVarP(&catalogOutformat, "out-format", "f", "json", "Format of output file or stdout (yaml or json)")
 	Query.Flags().DurationVarP(&catalogWaitFor, "wait", "w", 60*time.Second, "Wait for this long for resources to be discovered")
 	Catalog.AddCommand(Query)
 	Root.AddCommand(Catalog)

cmd/playbook.go

@@ -203,16 +203,16 @@ var Run = &cobra.Command{
 
 func saveOutput(object any, file string, format string) {
 	var out string
-	if outFormat == "yaml" {
+	if format == "yaml" {
 		b, _ := yaml.Marshal(object)
 		out = string(b)
 	} else {
 		b, _ := json.MarshalIndent(object, "", "  ")
 		out = string(b)
 	}
 
-	if outfile != "" {
-		_ = os.WriteFile(outfile, []byte(out), 0600)
+	if file != "" {
+		_ = os.WriteFile(file, []byte(out), 0600)
 	} else {
 		fmt.Println(out)
 	}

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/containrrr/shoutrrr v0.8.0
 	github.com/fergusstrange/embedded-postgres v1.25.0 // indirect
 	github.com/flanksource/commons v1.29.10
-	github.com/flanksource/duty v1.0.658
+	github.com/flanksource/duty v1.0.660
 	github.com/flanksource/gomplate/v3 v3.24.32
 	github.com/flanksource/kopper v1.0.10
 	github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2

go.sum

@@ -877,8 +877,8 @@ github.com/flanksource/artifacts v1.0.14 h1:Vv70bccsae0MwGaf/uSPp34J5V1/PyKfct9z
 github.com/flanksource/artifacts v1.0.14/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70=
 github.com/flanksource/commons v1.29.10 h1:T/S95Pl8kASEFvQjQ7fJjTUqeVdhxQXg1vfkULTYFJQ=
 github.com/flanksource/commons v1.29.10/go.mod h1:iTbrXOSp3Spv570Nly97D/U9cQjLZoVlmWCXqWzsvRU=
-github.com/flanksource/duty v1.0.658 h1:JRd+svAimAYDG/1gYf5+/clFmhXSYmjC9fCHT9jHpao=
-github.com/flanksource/duty v1.0.658/go.mod h1:Oj9zIX94CR2U+nmnt97gNLMrsBWILyIhIBeJynIIgqE=
+github.com/flanksource/duty v1.0.660 h1:h9JO4lJMB+u6cS7L+KOz6bJvREAJ78QNaNVeMPWmTI8=
+github.com/flanksource/duty v1.0.660/go.mod h1:Nl00WRVSA1RYN7+3Tg5vjbtCM9v2zZxjZMU4jnGP+s4=
 github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc=
 github.com/flanksource/gomplate/v3 v3.24.32 h1:MILauVcjIqBit4nXB5UCN/LZ2z+fiMb8n1Klwjci1Tg=
 github.com/flanksource/gomplate/v3 v3.24.32/go.mod h1:FdQHxnyrBSmT5zNJTDq08oXxD+eOqti4ERanSoDmQAU=

playbook/runner/gitops.go

@@ -1,56 +1,13 @@
 package runner
 
 import (
-	"encoding/json"
-	"fmt"
-
-	"gopkg.in/yaml.v3"
-
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-
 	"github.com/flanksource/duty/context"
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/query"
 	v1 "github.com/flanksource/incident-commander/api/v1"
 )
 
-type GitOpsEnvKustomize struct {
-	Path string `json:"path"`
-}
-
-func (t *GitOpsEnvKustomize) AsMap() map[string]any {
-	return map[string]any{
-		"path": t.Path,
-	}
-}
-
-type GitOpsEnvGit struct {
-	File   string `json:"file"`
-	URL    string `json:"url"`
-	Branch string `json:"branch"`
-}
-
-func (t *GitOpsEnvGit) AsMap() map[string]any {
-	return map[string]any{
-		"file":   t.File,
-		"url":    t.URL,
-		"branch": t.Branch,
-	}
-}
-
-type GitOpsEnv struct {
-	Git       GitOpsEnvGit       `json:"git"`
-	Kustomize GitOpsEnvKustomize `json:"kustomize"`
-}
-
-func (t *GitOpsEnv) AsMap() map[string]any {
-	return map[string]any{
-		"git":       t.Git.AsMap(),
-		"kustomize": t.Kustomize.AsMap(),
-	}
-}
-
-func getGitOpsTemplateVars(ctx context.Context, run models.PlaybookRun, actions []v1.PlaybookAction) (*GitOpsEnv, error) {
+func getGitOpsTemplateVars(ctx context.Context, run models.PlaybookRun, actions []v1.PlaybookAction) (*query.GitOpsSource, error) {
 	if run.ConfigID == nil {
 		return nil, nil
 	}
@@ -67,43 +24,6 @@ func getGitOpsTemplateVars(ctx context.Context, run models.PlaybookRun, actions
 		return nil, nil
 	}
 
-	ci, err := query.GetCachedConfig(ctx, run.ConfigID.String())
-	if err != nil {
-		return nil, err
-	}
-	_ = ci
-
-	var gitOpsEnv GitOpsEnv
-
-	gitRepos := query.TraverseConfig(ctx, run.ConfigID.String(), "Kubernetes::Kustomization/Kubernetes::GitRepository", string(models.RelatedConfigTypeIncoming))
-	if len(gitRepos) > 0 && gitRepos[0].Config != nil {
-		var config map[string]any
-		if err := json.Unmarshal([]byte(*gitRepos[0].Config), &config); err != nil {
-			return nil, fmt.Errorf("failed to unmarshal config: %w", err)
-		}
-
-		gitOpsEnv.Git.URL, _, _ = unstructured.NestedString(config, "spec", "url")
-		gitOpsEnv.Git.Branch, _, _ = unstructured.NestedString(config, "spec", "ref", "branch")
-		_origin, _, _ := unstructured.NestedString(config, "metadata", "annotations", "config.kubernetes.io/origin")
-		if _origin != "" {
-			var origin map[string]any
-			if err := yaml.Unmarshal([]byte(_origin), &origin); err != nil {
-				ctx.Tracef("failed to unmarshal origin: %v", err)
-			} else if path, ok := origin["path"]; ok {
-				gitOpsEnv.Git.File = path.(string)
-			}
-		}
-	}
-
-	kustomization := query.TraverseConfig(ctx, run.ConfigID.String(), "Kubernetes::Kustomization", string(models.RelatedConfigTypeIncoming))
-	if len(kustomization) > 0 && kustomization[0].Config != nil {
-		var config map[string]any
-		if err := json.Unmarshal([]byte(*kustomization[0].Config), &config); err != nil {
-			return nil, fmt.Errorf("failed to unmarshal config: %w", err)
-		}
-
-		gitOpsEnv.Kustomize.Path, _, _ = unstructured.NestedString(config, "spec", "path")
-	}
-
-	return &gitOpsEnv, nil
+	source, err := query.GetGitOpsSource(ctx, *run.ConfigID)
+	return &source, err
 }

playbook/runner/template.go

@@ -111,14 +111,10 @@ func CreateTemplateEnv(ctx context.Context, playbook *models.Playbook, run *mode
 		return templateEnv, nil
 	}
 
-	var gitOpsEnvVar *GitOpsEnv
-	var err error
-	if gitOpsEnvVar, err = getGitOpsTemplateVars(ctx, *run, spec.Actions); err != nil {
+	if gitOpsEnvVar, err := getGitOpsTemplateVars(ctx, *run, spec.Actions); err != nil {
 		return templateEnv, oops.Wrapf(err, "failed to get gitops vars")
-	}
-
-	if gitOpsEnvVar != nil {
-		templateEnv.Env = collections.MergeMap(templateEnv.Env, gitOpsEnvVar.AsMap())
+	} else if gitOpsEnvVar != nil {
+		templateEnv.Env["git"] = gitOpsEnvVar.AsMap()
 	}
 
 	env := make(map[string]any)

rbac/objects.go

@@ -114,6 +114,7 @@ var dbResourceObjMap = map[string]string{
 	"notification_silences":              ObjectNotification,
 	"people_roles":                       ObjectDatabasePublic,
 	"people":                             ObjectPeople,
+	"permissions":                        ObjectDatabaseSystem,
 	"playbook_action_agent_data":         ObjectPlaybooks,
 	"playbook_approvals":                 ObjectPlaybooks,
 	"playbook_names":                     ObjectDatabasePublic,

upstream/upstream_test.go

@@ -123,6 +123,7 @@ func compareAgentEntities[T any](table string, upstreamDB *gorm.DB, agent agentW
 
 	Expect(len(upstream)).To(Equal(len(downstream)), fmt.Sprintf("expected %s to sync all items to upstream ", agent.name))
 
+	ignoreOpts = append(ignoreOpts, cmpopts.IgnoreUnexported(models.ConfigItem{}))
 	diff := cmp.Diff(upstream, downstream, ignoreOpts...)
 	Expect(diff).To(BeEmpty(), fmt.Sprintf("expected %s to sync correct items to upstream ", agent.name))
 }