feat(policies): Add granular policy_sets

server/core/config/raw/policies.go

@@ -2,15 +2,16 @@ package raw
 
 import (
 	validation "github.com/go-ozzo/ozzo-validation"
-	"github.com/hashicorp/go-version"
+	version "github.com/hashicorp/go-version"
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 )
 
 // PolicySets is the raw schema for repo-level atlantis.yaml config.
 type PolicySets struct {
-	Version    *string      `yaml:"conftest_version,omitempty" json:"conftest_version,omitempty"`
-	Owners     PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
-	PolicySets []PolicySet  `yaml:"policy_sets" json:"policy_sets"`
+	Version     *string      `yaml:"conftest_version,omitempty" json:"conftest_version,omitempty"`
+	Owners      PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	PolicySets  []PolicySet  `yaml:"policy_sets" json:"policy_sets"`
+	ReviewCount int          `yaml:"review_count,omitempty" json:"review_count,omitempty"`
 }
 
 func (p PolicySets) Validate() error {
@@ -27,10 +28,20 @@ func (p PolicySets) ToValid() valid.PolicySets {
 		policySets.Version, _ = version.NewVersion(*p.Version)
 	}
 
+	// Default number of required reviews for all policy sets should be 1.
+	policySets.ReviewCount = p.ReviewCount
+	if policySets.ReviewCount == 0 {
+		policySets.ReviewCount = 1
+	}
+
 	policySets.Owners = p.Owners.ToValid()
 
 	validPolicySets := make([]valid.PolicySet, 0)
 	for _, rawPolicySet := range p.PolicySets {
+		// Default to top-level review count if not specified.
+		if rawPolicySet.ReviewCount == 0 {
+			rawPolicySet.ReviewCount = policySets.ReviewCount
+		}
 		validPolicySets = append(validPolicySets, rawPolicySet.ToValid())
 	}
 	policySets.PolicySets = validPolicySets
@@ -57,16 +68,18 @@ func (o PolicyOwners) ToValid() valid.PolicyOwners {
 }
 
 type PolicySet struct {
-	Path   string       `yaml:"path" json:"path"`
-	Source string       `yaml:"source" json:"source"`
-	Name   string       `yaml:"name" json:"name"`
-	Owners PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	Path        string       `yaml:"path" json:"path"`
+	Source      string       `yaml:"source" json:"source"`
+	Name        string       `yaml:"name" json:"name"`
+	Owners      PolicyOwners `yaml:"owners,omitempty" json:"owners,omitempty"`
+	ReviewCount int          `yaml:"review_count,omitempty" json:"review_count,omitempty"`
 }
 
 func (p PolicySet) Validate() error {
 	return validation.ValidateStruct(&p,
 		validation.Field(&p.Name, validation.Required.Error("is required")),
 		validation.Field(&p.Owners),
+		validation.Field(&p.ReviewCount),
 		validation.Field(&p.Path, validation.Required.Error("is required")),
 		validation.Field(&p.Source, validation.In(valid.LocalPolicySet, valid.GithubPolicySet).Error("only 'local' and 'github' source types are supported")),
 	)
@@ -78,6 +91,7 @@ func (p PolicySet) ToValid() valid.PolicySet {
 	policySet.Name = p.Name
 	policySet.Path = p.Path
 	policySet.Source = p.Source
+	policySet.ReviewCount = p.ReviewCount
 	policySet.Owners = p.Owners.ToValid()
 
 	return policySet

server/core/config/valid/policies.go

@@ -3,7 +3,7 @@ package valid
 import (
 	"strings"
 
-	"github.com/hashicorp/go-version"
+	version "github.com/hashicorp/go-version"
 )
 
 const (
@@ -15,9 +15,10 @@ const (
 // PolicySet objects. PolicySets struct is used by PolicyCheck workflow to build
 // context to enforce policies.
 type PolicySets struct {
-	Version    *version.Version
-	Owners     PolicyOwners
-	PolicySets []PolicySet
+	Version     *version.Version
+	Owners      PolicyOwners
+	ReviewCount int
+	PolicySets  []PolicySet
 }
 
 type PolicyOwners struct {
@@ -26,28 +27,36 @@ type PolicyOwners struct {
 }
 
 type PolicySet struct {
-	Source string
-	Path   string
-	Name   string
-	Owners PolicyOwners
+	Source      string
+	Path        string
+	Name        string
+	ReviewCount int
+	Owners      PolicyOwners
 }
 
 func (p *PolicySets) HasPolicies() bool {
 	return len(p.PolicySets) > 0
 }
 
+// Check if any level of policy owners includes teams
 func (p *PolicySets) HasTeamOwners() bool {
-	return len(p.Owners.Teams) > 0
+	hasTeamOwners := len(p.Owners.Teams) > 0
+	for _, policySet := range p.PolicySets {
+		if len(policySet.Owners.Teams) > 0 {
+			hasTeamOwners = true
+		}
+	}
+	return hasTeamOwners
 }
 
-func (p *PolicySets) IsOwner(username string, userTeams []string) bool {
-	for _, uname := range p.Owners.Users {
+func (o *PolicyOwners) IsOwner(username string, userTeams []string) bool {
+	for _, uname := range o.Users {
 		if strings.EqualFold(uname, username) {
 			return true
 		}
 	}
 
-	for _, orgTeamName := range p.Owners.Teams {
+	for _, orgTeamName := range o.Teams {
 		for _, userTeamName := range userTeams {
 			if strings.EqualFold(orgTeamName, userTeamName) {
 				return true

server/core/db/boltdb.go

@@ -357,6 +357,7 @@ func (b *BoltDB) UpdatePullWithResults(pull models.PullRequest, newResults []com
 						res.ProjectName == proj.ProjectName {
 
 						proj.Status = res.PlanStatus()
+						proj.PolicyStatus = res.PolicyCheckApprovals
 						updatedExisting = true
 						break
 					}
@@ -483,9 +484,10 @@ func (b *BoltDB) writePullToBucket(bucket *bolt.Bucket, key []byte, pull models.
 
 func (b *BoltDB) projectResultToProject(p command.ProjectResult) models.ProjectStatus {
 	return models.ProjectStatus{
-		Workspace:   p.Workspace,
-		RepoRelDir:  p.RepoRelDir,
-		ProjectName: p.ProjectName,
-		Status:      p.PlanStatus(),
+		Workspace:    p.Workspace,
+		RepoRelDir:   p.RepoRelDir,
+		ProjectName:  p.ProjectName,
+		PolicyStatus: p.PolicyCheckApprovals,
+		Status:       p.PlanStatus(),
 	}
 }

server/core/redis/redis.go

@@ -329,6 +329,7 @@ func (r *RedisDB) UpdatePullWithResults(pull models.PullRequest, newResults []co
 					res.RepoRelDir == proj.RepoRelDir &&
 					res.ProjectName == proj.ProjectName {
 
+					proj.PolicyStatus = res.PolicyCheckApprovals
 					proj.Status = res.PlanStatus()
 					updatedExisting = true
 					break
@@ -399,9 +400,10 @@ func (r *RedisDB) pullKey(pull models.PullRequest) (string, error) {
 
 func (r *RedisDB) projectResultToProject(p command.ProjectResult) models.ProjectStatus {
 	return models.ProjectStatus{
-		Workspace:   p.Workspace,
-		RepoRelDir:  p.RepoRelDir,
-		ProjectName: p.ProjectName,
-		Status:      p.PlanStatus(),
+		Workspace:    p.Workspace,
+		RepoRelDir:   p.RepoRelDir,
+		ProjectName:  p.ProjectName,
+		PolicyStatus: p.PolicyCheckApprovals,
+		Status:       p.PlanStatus(),
 	}
 }

server/core/runtime/policy/conftest_client.go

@@ -7,16 +7,20 @@ import (
 	"runtime"
 	"strings"
 
+	"encoding/json"
+	"github.com/hashicorp/go-multierror"
 	version "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 	"github.com/runatlantis/atlantis/server/core/runtime/cache"
 	runtime_models "github.com/runatlantis/atlantis/server/core/runtime/models"
 	"github.com/runatlantis/atlantis/server/core/terraform"
 	"github.com/runatlantis/atlantis/server/events/command"
+	"github.com/runatlantis/atlantis/server/events/models"
 	"github.com/runatlantis/atlantis/server/logging"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
+	"regexp"
 )
 
 const (
@@ -163,46 +167,63 @@ func NewConfTestExecutorWorkflow(log logging.SimpleLogging, versionRootDir strin
 }
 
 func (c *ConfTestExecutorWorkflow) Run(ctx command.ProjectContext, executablePath string, envs map[string]string, workdir string, extraArgs []string) (string, error) {
-	policyArgs := []Arg{}
-	policySetNames := []string{}
 	ctx.Log.Debug("policy sets, %s ", ctx.PolicySets)
+
+	inputFile := filepath.Join(workdir, ctx.GetShowResultFileName())
+	var policySetResults []models.PolicySetResult
+	var combinedErr error
+
 	for _, policySet := range ctx.PolicySets.PolicySets {
-		path, err := c.SourceResolver.Resolve(policySet)
+		path, resolveErr := c.SourceResolver.Resolve(policySet)
 
 		// Let's not fail the whole step because of a single failure. Log and fail silently
-		if err != nil {
-			ctx.Log.Err("Error resolving policyset %s. err: %s", policySet.Name, err.Error())
+		if resolveErr != nil {
+			ctx.Log.Err("Error resolving policyset %s. err: %s", policySet.Name, resolveErr.Error())
 			continue
 		}
 
-		policyArg := NewPolicyArg(path)
-		policyArgs = append(policyArgs, policyArg)
-
-		policySetNames = append(policySetNames, policySet.Name)
-	}
+		args := ConftestTestCommandArgs{
+			PolicyArgs: []Arg{NewPolicyArg(path)},
+			ExtraArgs:  extraArgs,
+			InputFile:  inputFile,
+			Command:    executablePath,
+		}
 
-	inputFile := filepath.Join(workdir, ctx.GetShowResultFileName())
+		serializedArgs, _ := args.build()
+		cmdOutput, cmdErr := c.Exec.CombinedOutput(serializedArgs, envs, workdir)
+
+		passed := true
+		if cmdErr != nil {
+			// Since we're running conftest for each policyset, individual command errors should be concatenated.
+			if isValidConftestOutput(cmdOutput) {
+				combinedErr = multierror.Append(combinedErr, errors.New(fmt.Sprintf("policy_set: %s:\n  conftest:\n  %s", policySet.Name, "Some policies failed.")))
+			} else {
+				combinedErr = multierror.Append(combinedErr, errors.New(fmt.Sprintf("policy_set: %s:\n  conftest:\n  %s", policySet.Name, cmdOutput)))
+			}
+			passed = false
+		}
 
-	args := ConftestTestCommandArgs{
-		PolicyArgs: policyArgs,
-		ExtraArgs:  extraArgs,
-		InputFile:  inputFile,
-		Command:    executablePath,
+		policySetResults = append(policySetResults, models.PolicySetResult{
+			PolicySetName:   policySet.Name,
+			PolicySetOutput: cmdOutput,
+			Passed:          passed,
+		})
 	}
 
-	serializedArgs, err := args.build()
-
-	if err != nil {
-		ctx.Log.Warn("No policies have been configured")
+	if policySetResults == nil {
+		ctx.Log.Warn("No policies have been configured.")
 		return "", nil
 		// TODO: enable when we can pass policies in otherwise e2e tests with policy checks fail
 		// return "", errors.Wrap(err, "building args")
 	}
 
-	initialOutput := fmt.Sprintf("Checking plan against the following policies: \n  %s\n", strings.Join(policySetNames, "\n  "))
-	cmdOutput, err := c.Exec.CombinedOutput(serializedArgs, envs, workdir)
+	marshaledStatus, err := json.Marshal(policySetResults)
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("Cannot marshal data into []PolicySetResult. Error: %w Data: %w", err, policySetResults))
+	}
+	output := string(marshaledStatus)
 
-	return c.sanitizeOutput(inputFile, initialOutput+cmdOutput), err
+	return c.sanitizeOutput(inputFile, output), combinedErr
 
 }
 
@@ -255,3 +276,13 @@ func getDefaultVersion() (*version.Version, error) {
 	}
 	return wrappedVersion, nil
 }
+
+// Checks if output from conftest is a valid output.
+func isValidConftestOutput(output string) bool {
+
+	r := regexp.MustCompile(`^(WARN|FAIL|\[)`)
+	if match := r.FindString(output); match != "" {
+		return true
+	}
+	return false
+}

server/events/approve_policies_command_runner.go

@@ -3,6 +3,7 @@ package events
 import (
 	"fmt"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/runatlantis/atlantis/server/events/command"
 	"github.com/runatlantis/atlantis/server/events/models"
 	"github.com/runatlantis/atlantis/server/events/vcs"
@@ -92,9 +93,10 @@ func (a *ApprovePoliciesCommandRunner) Run(ctx *command.Context, cmd *CommentCom
 }
 
 func (a *ApprovePoliciesCommandRunner) buildApprovePolicyCommandResults(ctx *command.Context, prjCmds []command.ProjectContext) (result command.Result) {
-	// Check if vcs user is in the owner list of the PolicySets. All projects
+	// Check if vcs user is in the top-level owner list of the PolicySets. All projects
 	// share the same Owners list at this time so no reason to iterate over each
 	// project.
+	var prjResults []command.ProjectResult
 	if len(prjCmds) > 0 {
 		teams := []string{}
 
@@ -107,18 +109,51 @@ func (a *ApprovePoliciesCommandRunner) buildApprovePolicyCommandResults(ctx *com
 			}
 			teams = append(teams, userTeams...)
 		}
+		isAdmin := prjCmds[0].PolicySets.Owners.IsOwner(ctx.User.Username, teams)
+
+		for _, prjCmd := range prjCmds {
+			var prjErrs error
+			var prjPolicyStatus []models.PolicySetApproval
+			// Grab policy set status for project
+			for _, prjPullStatus := range ctx.PullStatus.Projects {
+				if prjCmd.Workspace == prjPullStatus.Workspace &&
+					prjCmd.RepoRelDir == prjPullStatus.RepoRelDir &&
+					prjCmd.ProjectName == prjPullStatus.ProjectName {
+					prjPolicyStatus = prjPullStatus.PolicyStatus
+				}
+			}
 
-		if !prjCmds[0].PolicySets.IsOwner(ctx.User.Username, teams) {
-			result.Error = fmt.Errorf("contact policy owners to approve failing policies")
-			return
-		}
-	}
-
-	var prjResults []command.ProjectResult
+			for _, policySet := range prjCmd.PolicySets.PolicySets {
+				isOwner := policySet.Owners.IsOwner(ctx.User.Username, teams) || isAdmin
+				for i, policyStatus := range prjPolicyStatus {
+					if policySet.Name == policyStatus.PolicySetName {
+						if policyStatus.Approvals == 0 {
+							continue
+						}
+						if isOwner {
+							prjPolicyStatus[i].Approvals = policyStatus.Approvals + 1
+						} else {
+							prjErrs = multierror.Append(fmt.Errorf("Policy set: %s user %s is not a policy owner. Please contact policy owners to approve failing policies", policySet.Name, ctx.User.Username))
+						}
+						if prjPolicyStatus[i].Approvals != 0 {
+							prjErrs = multierror.Append(prjErrs, fmt.Errorf("Policy set: %s requires %d approvals, have %d.", policySet.Name, policySet.ReviewCount, (0-prjPolicyStatus[i].Approvals)))
+						}
+					}
+				}
+			}
 
-	for _, prjCmd := range prjCmds {
-		prjResult := a.prjCmdRunner.ApprovePolicies(prjCmd)
-		prjResults = append(prjResults, prjResult)
+			prjResult := command.ProjectResult{
+				Command:              command.PolicyCheck,
+				Failure:              "",
+				Error:                prjErrs,
+				PolicyCheckSuccess:   nil,
+				PolicyCheckApprovals: prjPolicyStatus,
+				RepoRelDir:           prjCmd.RepoRelDir,
+				Workspace:            prjCmd.Workspace,
+				ProjectName:          prjCmd.ProjectName,
+			}
+			prjResults = append(prjResults, prjResult)
+		}
 	}
 	result.ProjectResults = prjResults
 	return