Skip to content

Commit

Permalink
Introduce new statuses: SKIP, MANU, ERRO
Browse files Browse the repository at this point in the history
  • Loading branch information
@hypnoglow
hypnoglow committed Nov 5, 2021
1 parent cc619e5 commit 85e4b94
Show file tree
Hide file tree
Showing 11 changed files with 240 additions and 67 deletions.
32 changes: 17 additions & 15 deletions check/check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,12 @@ const (
WARN State = "WARN"
// INFO informational message
INFO State = "INFO"

// SKIP for when a check should be skipped.
SKIP = "skip"
SKIP State = "SKIP"
// MANU for when a check is manual.
MANU State = "MANU"
// ERRO for errors in tests.
ERRO State = "ERRO"

// MASTER a master node
MASTER NodeType = "master"
Expand All @@ -58,8 +61,11 @@ const (
// MANAGEDSERVICES a node to run managedservices from
MANAGEDSERVICES = "managedservices"

// MANUAL Check Type
MANUAL string = "manual"
// TypeSkip is skip check type.
TypeSkip = "skip"

// TypeManual is manual check type.
TypeManual = "manual"
)

// Check contains information about a recommendation in the
Expand Down Expand Up @@ -118,18 +124,18 @@ func (c *Check) run() State {
return c.State
}

// If check type is skip, force result to INFO
if c.Type == SKIP {
// If check type is skip, force result to SKIP
if c.Type == TypeSkip {
c.Reason = "Test marked as skip"
c.State = INFO
c.State = SKIP
glog.V(3).Info(c.Reason)
return c.State
}

// If check type is manual force result to WARN
if c.Type == MANUAL {
// If check type is manual, force result to MANU
if c.Type == TypeManual {
c.Reason = "Test marked as a manual test"
c.State = WARN
c.State = MANU
glog.V(3).Info(c.Reason)
return c.State
}
Expand Down Expand Up @@ -172,11 +178,7 @@ func (c *Check) run() State {

if err != nil {
c.Reason = err.Error()
if c.Scored {
c.State = FAIL
} else {
c.State = WARN
}
c.State = ERRO
glog.V(3).Info(c.Reason)
}

Expand Down
42 changes: 36 additions & 6 deletions check/check_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,27 @@ func TestCheck_Run(t *testing.T) {
}

testCases := []TestCase{
{name: "Manual check should WARN", check: Check{Type: MANUAL}, Expected: WARN},
{name: "Skip check should INFO", check: Check{Type: "skip"}, Expected: INFO},
{name: "Unscored check (with no type) should WARN on failure", check: Check{Scored: false}, Expected: WARN},
{
name: "Manual check should MANU",
check: Check{
Type: TypeManual,
},
Expected: MANU,
},
{
name: "Skip check should SKIP",
check: Check{
Type: TypeSkip,
},
Expected: SKIP,
},
{
name: "Unscored check (with no type) should WARN on failure",
check: Check{
Scored: false,
},
Expected: WARN,
},
{
name: "Unscored check that pass should PASS",
check: Check{
Expand All @@ -42,9 +60,21 @@ func TestCheck_Run(t *testing.T) {
},
Expected: PASS,
},

{name: "Check with no tests should WARN", check: Check{Scored: true}, Expected: WARN},
{name: "Scored check with empty tests should FAIL", check: Check{Scored: true, Tests: &tests{}}, Expected: FAIL},
{
name: "Check with no tests should WARN",
check: Check{
Scored: true,
},
Expected: WARN,
},
{
name: "Scored check with empty tests should FAIL",
check: Check{
Scored: true,
Tests: &tests{},
},
Expected: FAIL,
},
{
name: "Scored check that doesn't pass should FAIL",
check: Check{
Expand Down
44 changes: 34 additions & 10 deletions check/controls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ type Group struct {
Fail int `json:"fail"`
Warn int `json:"warn"`
Info int `json:"info"`
Skip int `json:"skip"`
Manu int `json:"manu"`
Erro int `json:"erro"`
Text string `json:"desc"`
Checks []*Check `json:"results"`
}
Expand All @@ -74,6 +77,9 @@ type Summary struct {
Fail int `json:"total_fail"`
Warn int `json:"total_warn"`
Info int `json:"total_info"`
Skip int `json:"total_skip"`
Manu int `json:"total_manu"`
Erro int `json:"total_erro"`
}

// Predicate a predicate on the given Group and Check arguments.
Expand All @@ -99,7 +105,7 @@ func NewControls(t NodeType, in []byte, detectedVersion string) (*Controls, erro
func (controls *Controls) RunChecks(runner Runner, filter Predicate, skipIDMap map[string]bool) Summary {
var g []*Group
m := make(map[string]*Group)
controls.Summary.Pass, controls.Summary.Fail, controls.Summary.Warn, controls.Info = 0, 0, 0, 0
controls.Summary = Summary{}

for _, group := range controls.Groups {
for _, check := range group.Checks {
Expand All @@ -111,8 +117,8 @@ func (controls *Controls) RunChecks(runner Runner, filter Predicate, skipIDMap m
_, groupSkippedViaCmd := skipIDMap[group.ID]
_, checkSkippedViaCmd := skipIDMap[check.ID]

if group.Type == SKIP || groupSkippedViaCmd || checkSkippedViaCmd {
check.Type = SKIP
if group.Type == TypeSkip || groupSkippedViaCmd || checkSkippedViaCmd {
check.Type = TypeSkip
}

state := runner.Run(check)
Expand Down Expand Up @@ -158,8 +164,14 @@ func (controls *Controls) JUnit() ([]byte, error) {
suite := reporters.JUnitTestSuite{
Name: controls.Text,
TestCases: []reporters.JUnitTestCase{},
Tests: controls.Summary.Pass + controls.Summary.Fail + controls.Summary.Info + controls.Summary.Warn,
Failures: controls.Summary.Fail,
Tests: controls.Summary.Pass +
controls.Summary.Fail +
controls.Summary.Info +
controls.Summary.Warn +
controls.Summary.Skip +
controls.Summary.Manu +
controls.Summary.Erro,
Failures: controls.Summary.Fail,
}
for _, g := range controls.Groups {
for _, check := range g.Checks {
Expand All @@ -179,11 +191,10 @@ func (controls *Controls) JUnit() ([]byte, error) {
}

switch check.State {
case FAIL:
case FAIL, ERRO:
tc.FailureMessage = &reporters.JUnitFailureMessage{Message: check.Remediation}
case WARN, INFO:
// WARN and INFO are two different versions of skipped tests. Either way it would be a false positive/negative to report
// it any other way.
case WARN, INFO, SKIP, MANU:
// Different versions of skipped tests. It would be a false positive/negative to report it any other way.
tc.Skipped = &reporters.JUnitSkipped{}
case PASS:
default:
Expand Down Expand Up @@ -226,7 +237,7 @@ func (controls *Controls) ASFF() ([]*securityhub.AwsSecurityFinding, error) {
tf := ti.Format(time.RFC3339)
for _, g := range controls.Groups {
for _, check := range g.Checks {
if check.State == FAIL || check.State == WARN {
if check.State == FAIL || check.State == WARN || check.State == MANU || check.State == ERRO {
// ASFF ProductFields['Actual result'] can't be longer than 1024 characters
actualValue := check.ActualValue
remediation := check.Remediation
Expand Down Expand Up @@ -292,6 +303,7 @@ func getConfig(name string) (string, error) {
}
return r, nil
}

func summarize(controls *Controls, state State) {
switch state {
case PASS:
Expand All @@ -302,6 +314,12 @@ func summarize(controls *Controls, state State) {
controls.Summary.Warn++
case INFO:
controls.Summary.Info++
case SKIP:
controls.Summary.Skip++
case MANU:
controls.Summary.Manu++
case ERRO:
controls.Summary.Erro++
default:
glog.Warningf("Unrecognized state %s", state)
}
Expand All @@ -317,6 +335,12 @@ func summarizeGroup(group *Group, state State) {
group.Warn++
case INFO:
group.Info++
case SKIP:
group.Skip++
case MANU:
group.Manu++
case ERRO:
group.Erro++
default:
glog.Warningf("Unrecognized state %s", state)
}
Expand Down
41 changes: 34 additions & 7 deletions check/controls_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,10 @@ groups:
controls.RunChecks(normalRunner, allChecks, skipMap)

G1 := controls.Groups[0]
assertEqualGroupSummary(t, 0, 0, 3, 0, G1)
assertEqualGroupSummary(t, 0, 0, 0, 0, 3, 0, 0, G1)

G2 := controls.Groups[1]
assertEqualGroupSummary(t, 0, 0, 2, 0, G2)
assertEqualGroupSummary(t, 0, 0, 0, 0, 2, 0, 0, G2)
})
}

Expand Down Expand Up @@ -163,7 +163,7 @@ groups:
controls.RunChecks(normalRunner, allChecks, emptySkipList)

G1 := controls.Groups[0]
assertEqualGroupSummary(t, 0, 0, 1, 0, G1)
assertEqualGroupSummary(t, 0, 0, 0, 0, 1, 0, 0, G1)
})
}

Expand Down Expand Up @@ -214,7 +214,7 @@ groups:
G1 := controls.Groups[0]
assert.Equal(t, "G1", G1.ID)
assert.Equal(t, "G1/C1", G1.Checks[0].ID)
assertEqualGroupSummary(t, 1, 0, 0, 0, G1)
assertEqualGroupSummary(t, 1, 0, 0, 0, 0, 0, 0, G1)
// and
G2 := controls.Groups[1]
assert.Equal(t, "G2", G2.ID)
Expand All @@ -225,12 +225,15 @@ groups:
assert.Equal(t, "SomeSampleFlag=true", G2.Checks[0].Tests.TestItems[0].Flag)
assert.Equal(t, "Edit the config file /this/is/a/file/path and set SomeSampleFlag to true.\n", G2.Checks[0].Remediation)
assert.Equal(t, true, G2.Checks[0].Scored)
assertEqualGroupSummary(t, 0, 1, 0, 0, G2)
assertEqualGroupSummary(t, 0, 1, 0, 0, 0, 0, 0, G2)
// and
assert.Equal(t, 1, controls.Summary.Pass)
assert.Equal(t, 1, controls.Summary.Fail)
assert.Equal(t, 0, controls.Summary.Info)
assert.Equal(t, 0, controls.Summary.Warn)
assert.Equal(t, 0, controls.Summary.Skip)
assert.Equal(t, 0, controls.Summary.Manu)
assert.Equal(t, 0, controls.Summary.Erro)
// and
runner.AssertExpectations(t)
})
Expand Down Expand Up @@ -267,6 +270,9 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
Pass: 100,
Warn: 101,
Info: 102,
Skip: 0,
Manu: 0,
Erro: 0,
},
Groups: []*Group{
{
Expand All @@ -283,7 +289,7 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
</testcase>
</testsuite>`),
}, {
desc: "Warn and Info are considered skips and failed tests properly reported",
desc: "WARN, INFO, SKIP, MANU are considered skips and failed tests properly reported",
input: &Controls{
Groups: []*Group{
{
Expand All @@ -293,6 +299,9 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
{ID: "check2id", Text: "check2text", State: INFO},
{ID: "check3id", Text: "check3text", State: WARN},
{ID: "check4id", Text: "check4text", State: FAIL},
{ID: "check5id", Text: "check5text", State: SKIP},
{ID: "check6id", Text: "check6text", State: MANU},
{ID: "check7id", Text: "check7text", State: ERRO},
},
},
},
Expand All @@ -313,6 +322,18 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
<failure type=""></failure>
<system-out>{&#34;test_number&#34;:&#34;check4id&#34;,&#34;test_desc&#34;:&#34;check4text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;FAIL&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
</testcase>
<testcase name="check5id check5text" classname="" time="0">
<skipped></skipped>
<system-out>{&#34;test_number&#34;:&#34;check5id&#34;,&#34;test_desc&#34;:&#34;check5text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;SKIP&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
</testcase>
<testcase name="check6id check6text" classname="" time="0">
<skipped></skipped>
<system-out>{&#34;test_number&#34;:&#34;check6id&#34;,&#34;test_desc&#34;:&#34;check6text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;MANU&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
</testcase>
<testcase name="check7id check7text" classname="" time="0">
<failure type=""></failure>
<system-out>{&#34;test_number&#34;:&#34;check7id&#34;,&#34;test_desc&#34;:&#34;check7text&#34;,&#34;audit&#34;:&#34;&#34;,&#34;AuditEnv&#34;:&#34;&#34;,&#34;AuditConfig&#34;:&#34;&#34;,&#34;type&#34;:&#34;&#34;,&#34;remediation&#34;:&#34;&#34;,&#34;test_info&#34;:null,&#34;status&#34;:&#34;ERRO&#34;,&#34;actual_value&#34;:&#34;&#34;,&#34;scored&#34;:false,&#34;IsMultiple&#34;:false,&#34;expected_result&#34;:&#34;&#34;}</system-out>
</testcase>
</testsuite>`),
},
}
Expand Down Expand Up @@ -355,12 +376,15 @@ func TestControls_JUnitIncludesJSON(t *testing.T) {
}
}

func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn int, actual *Group) {
func assertEqualGroupSummary(t *testing.T, pass, fail, info, warn, skip, manu, erro int, actual *Group) {
t.Helper()
assert.Equal(t, pass, actual.Pass)
assert.Equal(t, fail, actual.Fail)
assert.Equal(t, info, actual.Info)
assert.Equal(t, warn, actual.Warn)
assert.Equal(t, skip, actual.Skip)
assert.Equal(t, manu, actual.Manu)
assert.Equal(t, erro, actual.Erro)
}

func TestControls_ASFF(t *testing.T) {
Expand Down Expand Up @@ -388,6 +412,9 @@ func TestControls_ASFF(t *testing.T) {
Pass: 100,
Warn: 101,
Info: 102,
Skip: 0,
Manu: 0,
Erro: 0,
},
Groups: []*Group{
{
Expand Down
Loading

0 comments on commit 85e4b94

Please sign in to comment.