diff --git a/cmd/playbook.go b/cmd/playbook.go index fb75807d2..fb6902170 100644 --- a/cmd/playbook.go +++ b/cmd/playbook.go @@ -143,7 +143,7 @@ var Run = &cobra.Command{ var action *v1.PlaybookAction var step *models.PlaybookRunAction - action, step, err = runner.GetNextActionToRun(ctx, *p, *run) + action, step, err = runner.GetNextActionToRun(ctx, *run) if err != nil { logger.Fatalf(err.Error()) return @@ -173,7 +173,7 @@ var Run = &cobra.Command{ break } - action, _, err = runner.GetNextActionToRun(ctx, *p, *run) + action, _, err = runner.GetNextActionToRun(ctx, *run) if action != nil && action.Name == runAction.Name { ctx.Errorf("%v", ctx.Oops().Errorf("Action cycle detected for: %s", action.Name)) shutdown.ShutdownAndExit(1, "") diff --git a/db/playbooks.go b/db/playbooks.go index f3604e844..6cd57c6e4 100644 --- a/db/playbooks.go +++ b/db/playbooks.go @@ -28,6 +28,19 @@ func FindPlaybooksForEvent(ctx context.Context, eventClass, event string) ([]mod return playbooks, nil } +func FindPlaybookRun(ctx context.Context, id uuid.UUID) (*models.PlaybookRun, error) { + var p models.PlaybookRun + if err := ctx.DB().Where("id = ?", id).First(&p).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + + return nil, err + } + + return &p, nil +} + func FindPlaybook(ctx context.Context, id uuid.UUID) (*models.Playbook, error) { var p models.Playbook if err := ctx.DB().Where("id = ?", id).First(&p).Error; err != nil { diff --git a/go.mod b/go.mod index 309e1cc4f..92df6246e 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/containrrr/shoutrrr v0.8.0 github.com/fergusstrange/embedded-postgres v1.25.0 // indirect github.com/flanksource/commons v1.29.10 - github.com/flanksource/duty v1.0.666 + github.com/flanksource/duty v1.0.667 github.com/flanksource/gomplate/v3 v3.24.32 github.com/flanksource/kopper v1.0.10 github.com/gomarkdown/markdown v0.0.0-20240419095408-642f0ee99ae2 diff --git a/go.sum b/go.sum index fc0f607eb..0d5da2599 100644 --- a/go.sum +++ b/go.sum @@ -877,8 +877,8 @@ github.com/flanksource/artifacts v1.0.14 h1:Vv70bccsae0MwGaf/uSPp34J5V1/PyKfct9z github.com/flanksource/artifacts v1.0.14/go.mod h1:qHVCnQu5k50aWNJ5UhpcAKEl7pAzqUrFFKGSm147G70= github.com/flanksource/commons v1.29.10 h1:T/S95Pl8kASEFvQjQ7fJjTUqeVdhxQXg1vfkULTYFJQ= github.com/flanksource/commons v1.29.10/go.mod h1:iTbrXOSp3Spv570Nly97D/U9cQjLZoVlmWCXqWzsvRU= -github.com/flanksource/duty v1.0.666 h1:doJ1OBKXHtb9Tlslh/LO7PSoVuqsl4R6B9cuGP+2CWw= -github.com/flanksource/duty v1.0.666/go.mod h1:/dIt7bXnQlVtVikTu1TY1syEcuGCSUBnKyKlxVR5sDw= +github.com/flanksource/duty v1.0.667 h1:um6ppCnvQF3roIduY1n6S9uPv/YkEEicPhzeFBbDzck= +github.com/flanksource/duty v1.0.667/go.mod h1:/dIt7bXnQlVtVikTu1TY1syEcuGCSUBnKyKlxVR5sDw= github.com/flanksource/gomplate/v3 v3.20.4/go.mod h1:27BNWhzzSjDed1z8YShO6W+z6G9oZXuxfNFGd/iGSdc= github.com/flanksource/gomplate/v3 v3.24.32 h1:MILauVcjIqBit4nXB5UCN/LZ2z+fiMb8n1Klwjci1Tg= github.com/flanksource/gomplate/v3 v3.24.32/go.mod h1:FdQHxnyrBSmT5zNJTDq08oXxD+eOqti4ERanSoDmQAU= diff --git a/playbook/actions/http.go b/playbook/actions/http.go index 15b6e88f2..9d72120d0 100644 --- a/playbook/actions/http.go +++ b/playbook/actions/http.go @@ -34,7 +34,7 @@ func (c *HTTP) Run(ctx context.Context, action v1.HTTPAction) (*HTTPResult, erro if _, err := url.Parse(connection.URL); err != nil { return nil, fmt.Errorf("failed to parse url(%q): %w", connection.URL, err) } - } else if connection == nil { + } else { connection = &models.Connection{ URL: action.URL, } diff --git a/playbook/approval.go b/playbook/approval.go index 071bd8ee1..c49e59cef 100644 --- a/playbook/approval.go +++ b/playbook/approval.go @@ -1,6 +1,7 @@ package playbook import ( + "encoding/json" "net/http" "github.com/flanksource/commons/collections" @@ -19,52 +20,45 @@ func HandlePlaybookRunApproval(c echo.Context) error { ctx := c.Request().Context().(context.Context) var ( - playbookID = c.Param("playbook_id") - runID = c.Param("run_id") + runID = c.Param("run_id") ) - playbookUUID, err := uuid.Parse(playbookID) - if err != nil { - return c.JSON(http.StatusBadRequest, dutyAPI.HTTPError{Err: err.Error(), Message: "invalid playbook id"}) - } - runUUID, err := uuid.Parse(runID) if err != nil { return c.JSON(http.StatusBadRequest, dutyAPI.HTTPError{Err: err.Error(), Message: "invalid run id"}) } - if err := ApproveRun(ctx, playbookUUID, runUUID); err != nil { + if err := ApproveRun(ctx, runUUID); err != nil { return dutyAPI.WriteError(c, err) } return c.JSON(http.StatusOK, dutyAPI.HTTPSuccess{Message: "playbook run approved"}) } -func ApproveRun(ctx context.Context, playbookID, runID uuid.UUID) error { - playbook, err := db.FindPlaybook(ctx, playbookID) +func ApproveRun(ctx context.Context, runID uuid.UUID) error { + run, err := db.FindPlaybookRun(ctx, runID) if err != nil { - return api.Errorf(api.EINTERNAL, "something went wrong while finding playbook(id=%s)", playbookID).WithDebugInfo("db.FindPlaybook(id=%s): %v", playbookID, err) - } else if playbook == nil { - return api.Errorf(api.ENOTFOUND, "playbook(id=%s) not found", playbookID) + return api.Errorf(api.EINTERNAL, "something went wrong while finding run (id=%s)", runID).WithDebugInfo("db.FindPlaybookRun(id=%s): %v", runID, err) + } else if run == nil { + return api.Errorf(api.ENOTFOUND, "playbook run (id=%s) not found", runID) } - return approveRun(ctx, playbook, runID) + return approveRun(ctx, run) } -func requiresApproval(playbook *models.Playbook) bool { - playbookV1, _ := v1.PlaybookFromModel(*playbook) - return playbookV1.Spec.Approval != nil && !playbookV1.Spec.Approval.Approvers.Empty() +func requiresApproval(spec v1.PlaybookSpec) bool { + return spec.Approval != nil && !spec.Approval.Approvers.Empty() } -func approveRun(ctx context.Context, playbook *models.Playbook, runID uuid.UUID) error { +func approveRun(ctx context.Context, run *models.PlaybookRun) error { approver := ctx.User() - playbookV1, err := v1.PlaybookFromModel(*playbook) - if err != nil { - return api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("v1.PlaybookFromModel: %v", err) + var spec v1.PlaybookSpec + if err := json.Unmarshal(run.Spec, &spec); err != nil { + return err } - if playbookV1.Spec.Approval == nil || playbookV1.Spec.Approval.Approvers.Empty() { + if spec.Approval == nil || spec.Approval.Approvers.Empty() { return api.Errorf(api.EINVALID, "this playbook does not require approval") } @@ -73,19 +67,19 @@ func approveRun(ctx context.Context, playbook *models.Playbook, runID uuid.UUID) } approval := models.PlaybookApproval{ - RunID: runID, + RunID: run.ID, } - if collections.Contains(playbookV1.Spec.Approval.Approvers.People, approver.Email) { + if collections.Contains(spec.Approval.Approvers.People, approver.Email) { approval.PersonID = &approver.ID } else { teams, err := db.GetTeamsForUser(ctx, approver.ID.String()) if err != nil { - return api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("db.GetTeamIDsForUser(id=%s): %v", approver.ID, err) + return api.Errorf(api.EINTERNAL, "something went wrong").WithDebugInfo("db.GetTeamsForUser(id=%s): %v", approver.ID, err) } for _, team := range teams { - if collections.Contains(playbookV1.Spec.Approval.Approvers.Teams, team.Name) { + if collections.Contains(spec.Approval.Approvers.Teams, team.Name) { approval.TeamID = &team.ID break } @@ -97,7 +91,7 @@ func approveRun(ctx context.Context, playbook *models.Playbook, runID uuid.UUID) } if err := db.SavePlaybookRunApproval(ctx, approval); err != nil { - return api.Errorf(api.EINTERNAL, "something went wrong while approving").WithDebugInfo("db.ApprovePlaybookRun(runID=%s, approverID=%s): %v", runID, approver.ID, err) + return api.Errorf(api.EINTERNAL, "something went wrong while approving").WithDebugInfo("db.SavePlaybookRunApproval(runID=%s, approverID=%s): %v", run.ID, approver.ID, err) } return nil diff --git a/playbook/controllers.go b/playbook/controllers.go index d9291ba6c..147dfaa92 100644 --- a/playbook/controllers.go +++ b/playbook/controllers.go @@ -43,7 +43,7 @@ func RegisterRoutes(e *echo.Echo) { runGroup := playbookGroup.Group("/run") runGroup.POST("", HandlePlaybookRun, rbac.Playbook(rbac.ActionRun)) runGroup.GET("/:id", HandleGetPlaybookRun, rbac.Playbook(rbac.ActionRead)) - runGroup.POST("/approve/:playbook_id/:run_id", HandlePlaybookRunApproval, rbac.Playbook(rbac.ActionApprove)) + runGroup.POST("/approve/:run_id", HandlePlaybookRunApproval, rbac.Playbook(rbac.ActionApprove)) } type RunResponse struct { diff --git a/playbook/events.go b/playbook/events.go index c8439d55c..2b71e78e7 100644 --- a/playbook/events.go +++ b/playbook/events.go @@ -175,6 +175,7 @@ func (t *playbookScheduler) Handle(ctx context.Context, event models.Event) erro run := models.PlaybookRun{ PlaybookID: p.ID, Status: models.PlaybookRunStatusPending, + Spec: p.Spec, } if playbook.Spec.Approval == nil || playbook.Spec.Approval.Approvers.Empty() { diff --git a/playbook/playbook.go b/playbook/playbook.go index 75203c3f0..8258da587 100644 --- a/playbook/playbook.go +++ b/playbook/playbook.go @@ -89,6 +89,10 @@ func Run(ctx context.Context, playbook *models.Playbook, req RunParams) (*models AgentID: req.AgentID, } + // The run gets its own copy of the spec and uses that throughout its lifecycle. + // Any change to the playbook spec while the run is in progress should not affect the run. + run.Spec = playbook.Spec + if ctx.User() != nil { run.CreatedBy = &ctx.User().ID } @@ -178,7 +182,7 @@ func saveRunAsConfigChange(ctx context.Context, playbook *models.Playbook, run m details := map[string]any{ "parameters": parameters, - "spec": playbook.Spec, + "spec": run.Spec, } detailsJSON, err := json.Marshal(details) if err != nil { @@ -211,14 +215,18 @@ func savePlaybookRun(ctx context.Context, playbook *models.Playbook, run *models defer tx.Rollback() ctx = ctx.WithDB(tx, ctx.Pool()) - if err := ctx.DB().Create(run).Error; err != nil { return ctx.Oops("db").Wrap(err) } - if requiresApproval(playbook) { + var spec v1.PlaybookSpec + if err := json.Unmarshal(run.Spec, &spec); err != nil { + return ctx.Oops().Wrap(err) + } + + if requiresApproval(spec) { // Attempt to auto approve run - if err := approveRun(ctx, playbook, run.ID); err != nil { + if err := ApproveRun(ctx, run.ID); err != nil { switch dutyAPI.ErrorCode(err) { case dutyAPI.EFORBIDDEN, dutyAPI.EINVALID: // ignore these errors diff --git a/playbook/run_consumer.go b/playbook/run_consumer.go index ac7189462..a86c6bb1f 100644 --- a/playbook/run_consumer.go +++ b/playbook/run_consumer.go @@ -290,6 +290,7 @@ func RunConsumer(ctx context.Context) (int, error) { if ctx.Properties().On(false, "playbook.scheduler.disabled") { return 0, nil } + var consumed = 0 err := ctx.Transaction(func(ctx context.Context, _ trace.Span) error { tx := ctx.FastDB() diff --git a/playbook/runner/agent.go b/playbook/runner/agent.go index afc5be585..8e5e7f8a5 100644 --- a/playbook/runner/agent.go +++ b/playbook/runner/agent.go @@ -95,7 +95,7 @@ func getActionForAgent(ctx context.Context, agent *models.Agent) (*ActionForAgen return nil, ctx.Oops().Wrapf(err, "failed to template env") } - spec, err := getActionSpec(playbook, step.Name) + spec, err := getActionSpec(run, step.Name) if err != nil { return nil, ctx.Oops().Wrap(err) } diff --git a/playbook/runner/runner.go b/playbook/runner/runner.go index 83a5c10b9..d1d1004cd 100644 --- a/playbook/runner/runner.go +++ b/playbook/runner/runner.go @@ -29,17 +29,18 @@ const ( Agent = "agent" ) -func GetNextActionToRun(ctx context.Context, playbook models.Playbook, run models.PlaybookRun) (action *v1.PlaybookAction, lastAction *models.PlaybookRunAction, err error) { - ctx.Logger.V(3).Infof("Getting next action to run for playbook %s run %s", playbook.Name, run.ID) - ctx = ctx.WithObject(playbook, run) - if validationErr, err := openapi.ValidatePlaybookSpec(playbook.Spec); err != nil { +func GetNextActionToRun(ctx context.Context, run models.PlaybookRun) (action *v1.PlaybookAction, lastAction *models.PlaybookRunAction, err error) { + ctx.Logger.V(3).Infof("getting next action for run %s", run.ID) + ctx = ctx.WithObject(run) + + if validationErr, err := openapi.ValidatePlaybookSpec(run.Spec); err != nil { return nil, nil, err } else if validationErr != nil { return nil, nil, validationErr } var playbookSpec v1.PlaybookSpec - if err := json.Unmarshal(playbook.Spec, &playbookSpec); err != nil { + if err := json.Unmarshal(run.Spec, &playbookSpec); err != nil { return nil, nil, ctx.Oops().Wrap(err) } @@ -86,14 +87,15 @@ func findNextActionWithFilter(actions []v1.PlaybookAction) *v1.PlaybookAction { return nil } -func getActionSpec(playbook *models.Playbook, name string) (*v1.PlaybookAction, error) { +func getActionSpec(run *models.PlaybookRun, name string) (*v1.PlaybookAction, error) { var spec v1.PlaybookSpec - if err := json.Unmarshal(playbook.Spec, &spec); err != nil { + if err := json.Unmarshal(run.Spec, &spec); err != nil { return nil, err } + for _, action := range spec.Actions { if action.Name == name { - action.PlaybookID = playbook.ID.String() + action.PlaybookID = run.PlaybookID.String() return &action, nil } } @@ -165,9 +167,12 @@ func ScheduleRun(ctx context.Context, run models.PlaybookRun) error { return ctx.Oops("db").Wrap(err) } - ctx = ctx.WithObject(playbook, run) + // Override the current spec with the run's spec + playbook.Spec = run.Spec + + ctx = ctx.WithObject(run) - action, lastRan, err := GetNextActionToRun(ctx, playbook, run) + action, lastRan, err := GetNextActionToRun(ctx, run) if err != nil { ctx.Tracef("Unable to get next action") return ctx.Oops().Wrap(err) @@ -176,7 +181,7 @@ func ScheduleRun(ctx context.Context, run models.PlaybookRun) error { return ctx.Oops("db").Wrap(run.End(ctx.DB())) } - ctx = ctx.WithObject(playbook, action, run) + ctx = ctx.WithObject(action, run) delayed, err := CheckDelay(ctx, playbook, run, action, lastRan) if err != nil { @@ -193,7 +198,7 @@ func ScheduleRun(ctx context.Context, run models.PlaybookRun) error { } var playbookSpec v1.PlaybookSpec - if err := json.Unmarshal(playbook.Spec, &playbookSpec); err != nil { + if err := json.Unmarshal(run.Spec, &playbookSpec); err != nil { return ctx.Oops().Wrap(err) } @@ -261,22 +266,22 @@ func RunAction(ctx context.Context, run *models.PlaybookRun, action *models.Play playbook, err := action.GetPlaybook(ctx.DB()) if err != nil { return err - } - if playbook == nil { + } else if playbook == nil { return ctx.Oops().Errorf("playbook not found") } - spec, err := v1.PlaybookFromModel(*playbook) - if err != nil { + var spec v1.PlaybookSpec + if err := json.Unmarshal([]byte(run.Spec), &spec); err != nil { return err } - ctx = ctx.WithObject(playbook, action, run) + + ctx = ctx.WithObject(action, run) ctx, span := ctx.StartSpan(fmt.Sprintf("playbook.%s", playbook.Name)) defer span.End() + if err := TemplateAndExecuteAction(ctx, spec, playbook, run, action); err != nil { if e, ok := oops.AsOops(err); ok { if lo.Contains(e.Tags(), "db") { - // DB errors are retryable return err } @@ -297,10 +302,10 @@ func RunAction(ctx context.Context, run *models.PlaybookRun, action *models.Play } // TemplateAndExecuteAction executes the given playbook action after templating it. -func TemplateAndExecuteAction(ctx context.Context, spec v1.Playbook, playbook *models.Playbook, run *models.PlaybookRun, action *models.PlaybookRunAction) error { +func TemplateAndExecuteAction(ctx context.Context, spec v1.PlaybookSpec, playbook *models.Playbook, run *models.PlaybookRun, action *models.PlaybookRunAction) error { ctx = ctx.WithObject(playbook, run, action) - step, found := lo.Find(spec.Spec.Actions, func(i v1.PlaybookAction) bool { return i.Name == action.Name }) + step, found := lo.Find(spec.Actions, func(i v1.PlaybookAction) bool { return i.Name == action.Name }) if !found { return ctx.Oops().Errorf("action '%s' not found", action.Name) } diff --git a/playbook/runner/template.go b/playbook/runner/template.go index f838f8f98..35e498746 100644 --- a/playbook/runner/template.go +++ b/playbook/runner/template.go @@ -28,7 +28,7 @@ func CreateTemplateEnv(ctx context.Context, playbook *models.Playbook, run *mode oops := oops.With(models.ErrorContext(playbook, run)...) var spec v1.PlaybookSpec - if err := json.Unmarshal(playbook.Spec, &spec); err != nil { + if err := json.Unmarshal(run.Spec, &spec); err != nil { return templateEnv, oops.Wrapf(err, "invalid playbook spec") }