diff --git a/pkg/runner/test_runner.go b/pkg/runner/test_runner.go index b62f5633..b47db202 100644 --- a/pkg/runner/test_runner.go +++ b/pkg/runner/test_runner.go @@ -16,6 +16,7 @@ import ( "github.com/facebookincubator/contest/pkg/cerrors" "github.com/facebookincubator/contest/pkg/config" + "github.com/facebookincubator/contest/pkg/event" "github.com/facebookincubator/contest/pkg/event/testevent" "github.com/facebookincubator/contest/pkg/logging" "github.com/facebookincubator/contest/pkg/statectx" @@ -329,23 +330,24 @@ func (tr *testRunner) waitStepRunners(ctx statectx.Context) error { tr.cond.Wait() } }() + var err error select { case <-swch: tr.log.Debugf("step runners finished") tr.mu.Lock() defer tr.mu.Unlock() - return tr.checkStepRunners() + err = tr.checkStepRunners() case <-time.After(tr.shutdownTimeout): tr.log.Errorf("step runners failed to shut down correctly") + tr.mu.Lock() + defer tr.mu.Unlock() // If there is a step with an error set, use that. - err := tr.checkStepRunners() + err = tr.checkStepRunners() // If there isn't, enumerate ones that were still running at the time. nrerr := &cerrors.ErrTestStepsNeverReturned{} if err == nil { err = nrerr } - tr.mu.Lock() - defer tr.mu.Unlock() for _, ss := range tr.steps { if ss.stepRunning { nrerr.StepNames = append(nrerr.StepNames, ss.sb.TestStepLabel) @@ -353,8 +355,16 @@ func (tr *testRunner) waitStepRunners(ctx statectx.Context) error { tr.safeCloseOutCh(ss) } } - return err } + // Emit step error events. + for _, ss := range tr.steps { + if ss.runErr != nil && ss.runErr != statectx.ErrPaused && ss.runErr != statectx.ErrCanceled { + if err := ss.emitEvent(EventTestError, nil, ss.runErr.Error()); err != nil { + tr.log.Errorf("failed to emit event: %s", err) + } + } + } + return err } // targetRunner runs one target through all the steps of the pipeline. @@ -393,16 +403,28 @@ loop: select { case ss.inCh <- ts.tgt: // Injected successfully. + err := ss.ev.Emit(testevent.Data{EventName: target.EventTargetIn, Target: ts.tgt}) tr.mu.Lock() ts.CurPhase = targetStepPhaseRun ss.numInjected++ + if err != nil { + ss.runErr = fmt.Errorf("failed to report target injection: %w", err) + ss.log.Errorf("%s", ss.runErr) + } tr.mu.Unlock() tr.cond.Signal() + if err != nil { + break loop + } case <-time.After(tr.stepInjectTimeout): tr.mu.Lock() ss.log.Errorf("timed out while injecting a target") ss.runErr = &cerrors.ErrTestTargetInjectionTimedOut{StepName: ss.sb.TestStepLabel} tr.mu.Unlock() + err := ss.ev.Emit(testevent.Data{EventName: target.EventTargetInErr, Target: ts.tgt}) + if err != nil { + ss.log.Errorf("failed to emit event: %s", err) + } break loop case <-ctx.Done(): log.Debugf("%s: canceled 1", ts) @@ -416,6 +438,15 @@ loop: break loop } log.Debugf("%s: result for %s recd", ts, ss) + var err error + if res == nil { + err = ss.emitEvent(target.EventTargetOut, ts.tgt, nil) + } else { + err = ss.emitEvent(target.EventTargetErr, ts.tgt, target.ErrPayload{Error: res.Error()}) + } + if err != nil { + ss.log.Errorf("failed to emit event: %s", err) + } tr.mu.Lock() ts.CurPhase = targetStepPhaseEnd ts.res = res @@ -464,20 +495,21 @@ func (tr *testRunner) runStepIfNeeded(ctx statectx.Context, ss *stepState) { go tr.stepReader(ctx, ss) } -// emitStepEvent emits an error event if step resulted in an error. -func (ss *stepState) emitStepEvent(tgt *target.Target, err error) error { - if err == nil { - return nil - } - payload, jmErr := json.Marshal(err.Error()) - if jmErr != nil { - return fmt.Errorf("failed to marshal event: %w", err) +// emitEvent emits the specified event with the specified JSON payload (if any). +func (ss *stepState) emitEvent(name event.Name, tgt *target.Target, payload interface{}) error { + var payloadJSON *json.RawMessage + if payload != nil { + payloadBytes, jmErr := json.Marshal(payload) + if jmErr != nil { + return fmt.Errorf("failed to marshal event: %w", jmErr) + } + pj := json.RawMessage(payloadBytes) + payloadJSON = &pj } - rm := json.RawMessage(payload) errEv := testevent.Data{ - EventName: EventTestError, + EventName: name, Target: tgt, - Payload: &rm, + Payload: payloadJSON, } return ss.ev.Emit(errEv) } @@ -491,7 +523,7 @@ func (tr *testRunner) stepRunner(ctx statectx.Context, ss *stepState) { ss.stepRunning = false ss.runErr = &cerrors.ErrTestStepPaniced{ StepName: ss.sb.TestStepLabel, - StackTrace: string(debug.Stack()), + StackTrace: fmt.Sprintf("%s / %s", r, debug.Stack()), } tr.mu.Unlock() tr.safeCloseOutCh(ss) @@ -499,11 +531,6 @@ func (tr *testRunner) stepRunner(ctx statectx.Context, ss *stepState) { }() chans := test.TestStepChannels{In: ss.inCh, Out: ss.outCh, Err: ss.errCh} runErr := ss.sb.TestStep.Run(ctx, chans, ss.sb.Parameters, ss.ev) - if err := ss.emitStepEvent(nil, runErr); err != nil { - if ss.runErr == nil { - ss.runErr = err - } - } tr.mu.Lock() ss.stepRunning = false if runErr != nil { @@ -548,11 +575,6 @@ func (tr *testRunner) reportTargetResult(ctx statectx.Context, ss *stepState, tg if err != nil { return err } - if res != nil { - if err := ss.emitStepEvent(tgt, res); err != nil { - return err - } - } select { case resCh <- res: break diff --git a/pkg/runner/test_runner_test.go b/pkg/runner/test_runner_test.go index 444b4676..d14f7b94 100644 --- a/pkg/runner/test_runner_test.go +++ b/pkg/runner/test_runner_test.go @@ -33,6 +33,7 @@ import ( "github.com/facebookincubator/contest/tests/plugins/teststeps/hanging" "github.com/facebookincubator/contest/tests/plugins/teststeps/noreturn" "github.com/facebookincubator/contest/tests/plugins/teststeps/panicstep" + "github.com/facebookincubator/contest/tests/plugins/teststeps/teststep" ) const ( @@ -69,6 +70,7 @@ func TestMain(m *testing.M) { {hanging.Name, hanging.New, hanging.Events}, {noreturn.Name, noreturn.New, noreturn.Events}, {panicstep.Name, panicstep.New, panicstep.Events}, + {teststep.Name, teststep.New, teststep.Events}, } { if err := pluginRegistry.RegisterTestStep(e.name, e.factory, e.events); err != nil { panic(fmt.Sprintf("could not register TestStep: %v", err)) @@ -156,11 +158,11 @@ func newStep(label, name string, params *test.TestStepParameters) test.TestStepB return *sb } -func newExampleStep(label string, failPct int, failTargets string, delayTargets string) test.TestStepBundle { - return newStep(label, example.Name, &test.TestStepParameters{ - example.FailPctParam: []test.Param{*test.NewParam(fmt.Sprintf("%d", failPct))}, - example.FailTargetsParam: []test.Param{*test.NewParam(failTargets)}, - example.DelayTargetsParam: []test.Param{*test.NewParam(delayTargets)}, +func newTestStep(label string, failPct int, failTargets string, delayTargets string) test.TestStepBundle { + return newStep(label, teststep.Name, &test.TestStepParameters{ + teststep.FailPctParam: []test.Param{*test.NewParam(fmt.Sprintf("%d", failPct))}, + teststep.FailTargetsParam: []test.Param{*test.NewParam(failTargets)}, + teststep.DelayTargetsParam: []test.Param{*test.NewParam(delayTargets)}, }) } @@ -197,17 +199,19 @@ func Test1Step1Success(t *testing.T) { _, err := runWithTimeout(t, tr, nil, nil, 1, 2*time.Second, []*target.Target{tgt("T1")}, []test.TestStepBundle{ - newExampleStep("Step 1", 0, "", ""), + newTestStep("Step 1", 0, "", ""), }, ) require.NoError(t, err) require.Equal(t, ` -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepRunningEvent]} -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepFinishedEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepRunningEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepFinishedEvent]} `, getStepEvents("")) require.Equal(t, ` -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleFinishedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestFinishedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetOut]} `, getTargetEvents("T1")) } @@ -218,18 +222,19 @@ func Test1Step1Fail(t *testing.T) { _, err := runWithTimeout(t, tr, nil, nil, 1, 2*time.Second, []*target.Target{tgt("T1")}, []test.TestStepBundle{ - newExampleStep("Step 1", 100, "", ""), + newTestStep("Step 1", 100, "", ""), }, ) require.NoError(t, err) require.Equal(t, ` -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepRunningEvent]} -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepFinishedEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepRunningEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepFinishedEvent]} `, getStepEvents("Step 1")) require.Equal(t, ` -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleFailedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestError &"\"target failed\""]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestFailedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetErr &"{\"Error\":\"target failed\"}"]} `, getTargetEvents("T1")) } @@ -240,22 +245,25 @@ func Test1Step1Success1Fail(t *testing.T) { _, err := runWithTimeout(t, tr, nil, nil, 1, 2*time.Second, []*target.Target{tgt("T1"), tgt("T2")}, []test.TestStepBundle{ - newExampleStep("Step 1", 0, "T1", "T2=100"), + newTestStep("Step 1", 0, "T1", "T2=100"), }, ) require.NoError(t, err) require.Equal(t, ` -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepRunningEvent]} -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepFinishedEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepRunningEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepFinishedEvent]} `, getStepEvents("")) require.Equal(t, ` -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleFailedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestError &"\"target failed\""]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestFailedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetErr &"{\"Error\":\"target failed\"}"]} `, getTargetEvents("T1")) require.Equal(t, ` -{[1 1 SimpleTest Step 1][Target{ID: "T2"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T2"} ExampleFinishedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TestFinishedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TargetOut]} `, getTargetEvents("T2")) } @@ -267,32 +275,36 @@ func Test3StepsNotReachedStepNotRun(t *testing.T) { _, err := runWithTimeout(t, tr, nil, nil, 1, 2*time.Second, []*target.Target{tgt("T1"), tgt("T2")}, []test.TestStepBundle{ - newExampleStep("Step 1", 0, "T1", ""), - newExampleStep("Step 2", 0, "T2", ""), - newExampleStep("Step 3", 0, "", ""), + newTestStep("Step 1", 0, "T1", ""), + newTestStep("Step 2", 0, "T2", ""), + newTestStep("Step 3", 0, "", ""), }, ) require.NoError(t, err) require.Equal(t, ` -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepRunningEvent]} -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepFinishedEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepRunningEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepFinishedEvent]} `, getStepEvents("Step 1")) require.Equal(t, ` -{[1 1 SimpleTest Step 2][(*Target)(nil) ExampleStepRunningEvent]} -{[1 1 SimpleTest Step 2][(*Target)(nil) ExampleStepFinishedEvent]} +{[1 1 SimpleTest Step 2][(*Target)(nil) TestStepRunningEvent]} +{[1 1 SimpleTest Step 2][(*Target)(nil) TestStepFinishedEvent]} `, getStepEvents("Step 2")) require.Equal(t, "\n\n", getStepEvents("Step 3")) require.Equal(t, ` -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleFailedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestError &"\"target failed\""]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestFailedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetErr &"{\"Error\":\"target failed\"}"]} `, getTargetEvents("T1")) require.Equal(t, ` -{[1 1 SimpleTest Step 1][Target{ID: "T2"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T2"} ExampleFinishedEvent]} -{[1 1 SimpleTest Step 2][Target{ID: "T2"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 2][Target{ID: "T2"} ExampleFailedEvent]} -{[1 1 SimpleTest Step 2][Target{ID: "T2"} TestError &"\"target failed\""]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TestFinishedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TargetOut]} +{[1 1 SimpleTest Step 2][Target{ID: "T2"} TargetIn]} +{[1 1 SimpleTest Step 2][Target{ID: "T2"} TestStartedEvent]} +{[1 1 SimpleTest Step 2][Target{ID: "T2"} TestFailedEvent]} +{[1 1 SimpleTest Step 2][Target{ID: "T2"} TargetErr &"{\"Error\":\"target failed\"}"]} `, getTargetEvents("T2")) } @@ -342,6 +354,7 @@ func TestStepPanics(t *testing.T) { require.Error(t, err) require.IsType(t, &cerrors.ErrTestStepPaniced{}, err) require.Equal(t, "\n\n", getTargetEvents("T1")) + require.Contains(t, getStepEvents("Step 1"), "step Step 1 paniced") } // A misbehaving step that closes its output channel. @@ -356,7 +369,13 @@ func TestStepClosesChannels(t *testing.T) { ) require.Error(t, err) require.IsType(t, &cerrors.ErrTestStepClosedChannels{}, err) - require.Equal(t, "\n\n", getTargetEvents("T1")) + require.Equal(t, ` +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetOut]} +`, getTargetEvents("T1")) + require.Equal(t, ` +{[1 1 SimpleTest Step 1][(*Target)(nil) TestError &"\"test step Step 1 closed output channels (api violation)\""]} +`, getStepEvents("Step 1")) } // A misbehaving step that yields a result for a target that does not exist. @@ -370,6 +389,13 @@ func TestStepYieldsResultForNonexistentTarget(t *testing.T) { }, ) require.Error(t, err) + require.Equal(t, ` +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetIn]} +`, getTargetEvents("T1")) + require.Equal(t, "\n\n", getTargetEvents("T1XXX")) + require.Equal(t, ` +{[1 1 SimpleTest Step 1][(*Target)(nil) TestError &"\"[#0 Step 1]: result for nonexistent target Target{ID: \\\"T1XXX\\\"} \\u003cnil\\u003e\""]} +`, getStepEvents("Step 1")) } // A misbehaving step that yields a result for a target that does not exist. @@ -382,7 +408,7 @@ func TestStepYieldsDuplicateResult(t *testing.T) { // TGood makes it past here unscathed and gets delayed in Step 2, // TDup also emerges fine at first but is then returned again, and that's bad. newStep("Step 1", badtargets.Name, nil), - newExampleStep("Step 2", 0, "", "TGood=100"), + newTestStep("Step 2", 0, "", "TGood=100"), }, ) require.Error(t, err) @@ -412,7 +438,7 @@ func TestStepYieldsResultForUnexpectedTarget(t *testing.T) { []*target.Target{tgt("T1"), tgt("T1XXX")}, []test.TestStepBundle{ // T1XXX fails here. - newExampleStep("Step 1", 0, "T1XXX", ""), + newTestStep("Step 1", 0, "T1XXX", ""), // Yet, a result for it is returned here, which we did not expect. newStep("Step 2", badtargets.Name, nil), }, @@ -432,9 +458,9 @@ func TestRandomizedMultiStep(t *testing.T) { _, err := runWithTimeout(t, tr, nil, nil, 1, 2*time.Second, targets, []test.TestStepBundle{ - newExampleStep("Step 1", 0, "", "*=10"), // All targets pass the first step, with a slight delay - newExampleStep("Step 2", 25, "", ""), // 25% don't make it past the second step - newExampleStep("Step 3", 25, "", "*=10"), // Another 25% fail at the third step + newTestStep("Step 1", 0, "", "*=10"), // All targets pass the first step, with a slight delay + newTestStep("Step 2", 25, "", ""), // 25% don't make it past the second step + newTestStep("Step 3", 25, "", "*=10"), // Another 25% fail at the third step }, ) require.NoError(t, err) @@ -443,12 +469,14 @@ func TestRandomizedMultiStep(t *testing.T) { for _, tgt := range targets { s1n := "Step 1" require.Equal(t, fmt.Sprintf(` -{[1 1 SimpleTest Step 1][Target{ID: "%s"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "%s"} ExampleFinishedEvent]} -`, tgt.ID, tgt.ID), +{[1 1 SimpleTest Step 1][Target{ID: "%s"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "%s"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "%s"} TestFinishedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "%s"} TargetOut]} +`, tgt.ID, tgt.ID, tgt.ID, tgt.ID), getEvents(&tgt.ID, &s1n)) s3n := "Step 3" - if strings.Contains(getEvents(&tgt.ID, &s3n), "ExampleFinishedEvent") { + if strings.Contains(getEvents(&tgt.ID, &s3n), "TestFinishedEvent") { numFinished++ } } @@ -465,10 +493,10 @@ func TestPauseResumeSimple(t *testing.T) { var resumeState []byte targets := []*target.Target{tgt("T1"), tgt("T2"), tgt("T3")} steps := []test.TestStepBundle{ - newExampleStep("Step 1", 0, "T1", ""), + newTestStep("Step 1", 0, "T1", ""), // T2 and T3 will be paused here, the step will be given time to finish. - newExampleStep("Step 2", 0, "", "T2=200,T3=200"), - newExampleStep("Step 3", 0, "", ""), + newTestStep("Step 2", 0, "", "T2=200,T3=200"), + newTestStep("Step 3", 0, "", ""), } { tr1 := newTestRunner() @@ -526,9 +554,9 @@ func TestPauseResumeSimple(t *testing.T) { // Don't use the same pointers ot make sure there is no reliance on that. []*target.Target{tgt("T1"), tgt("T2"), tgt("T3")}, []test.TestStepBundle{ - newExampleStep("Step 1", 0, "T1", ""), - newExampleStep("Step 2", 0, "", "T2=200,T3=200"), - newExampleStep("Step 3", 0, "", ""), + newTestStep("Step 1", 0, "T1", ""), + newTestStep("Step 2", 0, "", "T2=200,T3=200"), + newTestStep("Step 3", 0, "", ""), }, ) require.NoError(t, err) @@ -537,31 +565,38 @@ func TestPauseResumeSimple(t *testing.T) { // Steps 1 and 2 are executed entirely within the first runner instance // and never started in the second. require.Equal(t, ` -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepRunningEvent]} -{[1 1 SimpleTest Step 1][(*Target)(nil) ExampleStepFinishedEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepRunningEvent]} +{[1 1 SimpleTest Step 1][(*Target)(nil) TestStepFinishedEvent]} `, getStepEvents("Step 1")) require.Equal(t, ` -{[1 1 SimpleTest Step 2][(*Target)(nil) ExampleStepRunningEvent]} -{[1 1 SimpleTest Step 2][(*Target)(nil) ExampleStepFinishedEvent]} +{[1 1 SimpleTest Step 2][(*Target)(nil) TestStepRunningEvent]} +{[1 1 SimpleTest Step 2][(*Target)(nil) TestStepFinishedEvent]} `, getStepEvents("Step 2")) // Step 3 did not get to start in the first instance and ran in the second. require.Equal(t, ` -{[1 5 SimpleTest Step 3][(*Target)(nil) ExampleStepRunningEvent]} -{[1 5 SimpleTest Step 3][(*Target)(nil) ExampleStepFinishedEvent]} +{[1 5 SimpleTest Step 3][(*Target)(nil) TestStepRunningEvent]} +{[1 5 SimpleTest Step 3][(*Target)(nil) TestStepFinishedEvent]} `, getStepEvents("Step 3")) // T1 failed entirely within the first run. require.Equal(t, ` -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} ExampleFailedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestError &"\"target failed\""]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TestFailedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T1"} TargetErr &"{\"Error\":\"target failed\"}"]} `, getTargetEvents("T1")) // T2 and T3 ran in both. require.Equal(t, ` -{[1 1 SimpleTest Step 1][Target{ID: "T2"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 1][Target{ID: "T2"} ExampleFinishedEvent]} -{[1 1 SimpleTest Step 2][Target{ID: "T2"} ExampleStartedEvent]} -{[1 1 SimpleTest Step 2][Target{ID: "T2"} ExampleFinishedEvent]} -{[1 5 SimpleTest Step 3][Target{ID: "T2"} ExampleStartedEvent]} -{[1 5 SimpleTest Step 3][Target{ID: "T2"} ExampleFinishedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TargetIn]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TestStartedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TestFinishedEvent]} +{[1 1 SimpleTest Step 1][Target{ID: "T2"} TargetOut]} +{[1 1 SimpleTest Step 2][Target{ID: "T2"} TargetIn]} +{[1 1 SimpleTest Step 2][Target{ID: "T2"} TestStartedEvent]} +{[1 1 SimpleTest Step 2][Target{ID: "T2"} TestFinishedEvent]} +{[1 1 SimpleTest Step 2][Target{ID: "T2"} TargetOut]} +{[1 5 SimpleTest Step 3][Target{ID: "T2"} TargetIn]} +{[1 5 SimpleTest Step 3][Target{ID: "T2"} TestStartedEvent]} +{[1 5 SimpleTest Step 3][Target{ID: "T2"} TestFinishedEvent]} +{[1 5 SimpleTest Step 3][Target{ID: "T2"} TargetOut]} `, getTargetEvents("T2")) } diff --git a/plugins/teststeps/example/example.go b/plugins/teststeps/example/example.go index c56b150f..3cd7f2a7 100644 --- a/plugins/teststeps/example/example.go +++ b/plugins/teststeps/example/example.go @@ -8,9 +8,7 @@ package example import ( "fmt" "math/rand" - "strconv" "strings" - "time" "github.com/facebookincubator/contest/pkg/cerrors" "github.com/facebookincubator/contest/pkg/event" @@ -29,23 +27,17 @@ var log = logging.GetLogger("teststeps/" + strings.ToLower(Name)) // Params this step accepts. const ( - // A comma-delimited list of target IDs to fail on. - FailTargetsParam = "FailTargets" - // Alternatively, fail this percentage of targets at random. + // Fail this percentage of targets at random. FailPctParam = "FailPct" - // A comma-delimited list of target IDs to delay and by how much, ID=delay_ms. - DelayTargetsParam = "DelayTargets" ) // events that we may emit during the plugin's lifecycle. This is used in Events below. // Note that you don't normally need to emit start/finish/cancellation events as // these are emitted automatically by the framework. const ( - StartedEvent = event.Name("ExampleStartedEvent") - FinishedEvent = event.Name("ExampleFinishedEvent") - FailedEvent = event.Name("ExampleFailedEvent") - StepRunningEvent = event.Name("ExampleStepRunningEvent") - StepFinishedEvent = event.Name("ExampleStepFinishedEvent") + StartedEvent = event.Name("ExampleStartedEvent") + FinishedEvent = event.Name("ExampleFinishedEvent") + FailedEvent = event.Name("ExampleFailedEvent") ) // Events defines the events that a TestStep is allow to emit. Emitting an event @@ -56,9 +48,7 @@ var Events = []event.Name{StartedEvent, FinishedEvent, FailedEvent} // consumes Targets in input and pipes them to the output or error channel // with intermediate buffering. type Step struct { - failPct int64 - failTargets map[string]bool - delayTargets map[string]time.Duration + failPct int64 } // Name returns the name of the Step @@ -66,10 +56,7 @@ func (ts Step) Name() string { return Name } -func (ts *Step) shouldFail(t *target.Target, params test.TestStepParameters) bool { - if ts.failTargets[t.ID] { - return true - } +func (ts *Step) shouldFail(t *target.Target) bool { if ts.failPct > 0 { roll := rand.Int63n(101) return (roll <= ts.failPct) @@ -87,16 +74,7 @@ func (ts *Step) Run(ctx statectx.Context, ch test.TestStepChannels, params test. if err := ev.Emit(testevent.Data{EventName: StartedEvent, Target: target, Payload: nil}); err != nil { return fmt.Errorf("failed to emit start event: %v", err) } - delay := ts.delayTargets[target.ID] - if delay == 0 { - delay = ts.delayTargets["*"] - } - select { - case <-time.After(delay): - case <-ctx.Done(): - return statectx.ErrCanceled - } - if ts.shouldFail(target, params) { + if ts.shouldFail(target) { if err := ev.Emit(testevent.Data{EventName: FailedEvent, Target: target, Payload: nil}); err != nil { return fmt.Errorf("failed to emit finished event: %v", err) } @@ -108,38 +86,11 @@ func (ts *Step) Run(ctx statectx.Context, ch test.TestStepChannels, params test. } return nil } - if err := ev.Emit(testevent.Data{EventName: StepRunningEvent}); err != nil { - return fmt.Errorf("failed to emit failed event: %v", err) - } - res := teststeps.ForEachTarget(Name, ctx, ch, f) - if err := ev.Emit(testevent.Data{EventName: StepFinishedEvent}); err != nil { - return fmt.Errorf("failed to emit failed event: %v", err) - } - return res + return teststeps.ForEachTarget(Name, ctx, ch, f) } // ValidateParameters validates the parameters associated to the TestStep func (ts *Step) ValidateParameters(params test.TestStepParameters) error { - targetsToFail := params.GetOne(FailTargetsParam).String() - if len(targetsToFail) > 0 { - for _, t := range strings.Split(targetsToFail, ",") { - ts.failTargets[t] = true - } - } - targetsToDelay := params.GetOne(DelayTargetsParam).String() - if len(targetsToDelay) > 0 { - for _, e := range strings.Split(targetsToDelay, ",") { - kv := strings.Split(e, "=") - if len(kv) != 2 { - continue - } - v, err := strconv.Atoi(kv[1]) - if err != nil { - return fmt.Errorf("invalid FailTargets: %w", err) - } - ts.delayTargets[kv[0]] = time.Duration(v) * time.Millisecond - } - } if params.GetOne(FailPctParam).String() != "" { if pct, err := params.GetInt(FailPctParam); err == nil { ts.failPct = pct @@ -163,10 +114,7 @@ func (ts *Step) CanResume() bool { // New initializes and returns a new ExampleTestStep. func New() test.TestStep { - return &Step{ - failTargets: make(map[string]bool), - delayTargets: make(map[string]time.Duration), - } + return &Step{} } // Load returns the name, factory and events which are needed to register the step. diff --git a/tests/plugins/teststeps/teststep/teststep.go b/tests/plugins/teststeps/teststep/teststep.go new file mode 100644 index 00000000..32db0907 --- /dev/null +++ b/tests/plugins/teststeps/teststep/teststep.go @@ -0,0 +1,160 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +package teststep + +import ( + "fmt" + "math/rand" + "strconv" + "strings" + "time" + + "github.com/facebookincubator/contest/pkg/cerrors" + "github.com/facebookincubator/contest/pkg/event" + "github.com/facebookincubator/contest/pkg/event/testevent" + "github.com/facebookincubator/contest/pkg/statectx" + "github.com/facebookincubator/contest/pkg/target" + "github.com/facebookincubator/contest/pkg/test" + "github.com/facebookincubator/contest/plugins/teststeps" +) + +var Name = "Test" + +const ( + // A comma-delimited list of target IDs to fail on. + FailTargetsParam = "FailTargets" + // Alternatively, fail this percentage of targets at random. + FailPctParam = "FailPct" + // A comma-delimited list of target IDs to delay and by how much, ID=delay_ms. + DelayTargetsParam = "DelayTargets" +) + +const ( + StartedEvent = event.Name("TestStartedEvent") + FinishedEvent = event.Name("TestFinishedEvent") + FailedEvent = event.Name("TestFailedEvent") + StepRunningEvent = event.Name("TestStepRunningEvent") + StepFinishedEvent = event.Name("TestStepFinishedEvent") +) + +var Events = []event.Name{StartedEvent, FinishedEvent, FailedEvent, StepRunningEvent, StepFinishedEvent} + +type Step struct { + failPct int64 + failTargets map[string]bool + delayTargets map[string]time.Duration +} + +// Name returns the name of the Step +func (ts Step) Name() string { + return Name +} + +func (ts *Step) shouldFail(t *target.Target, params test.TestStepParameters) bool { + if ts.failTargets[t.ID] { + return true + } + if ts.failPct > 0 { + roll := rand.Int63n(101) + return (roll <= ts.failPct) + } + return false +} + +// Run executes the example step. +func (ts *Step) Run(ctx statectx.Context, ch test.TestStepChannels, params test.TestStepParameters, ev testevent.Emitter) error { + f := func(ctx statectx.Context, target *target.Target) error { + // Sleep to ensure argetIn fires first. This simplifies test assertions. + time.Sleep(10 * time.Millisecond) + if err := ev.Emit(testevent.Data{EventName: StartedEvent, Target: target, Payload: nil}); err != nil { + return fmt.Errorf("failed to emit start event: %v", err) + } + delay := ts.delayTargets[target.ID] + if delay == 0 { + delay = ts.delayTargets["*"] + } + select { + case <-time.After(delay): + case <-ctx.Done(): + return statectx.ErrCanceled + } + if ts.shouldFail(target, params) { + if err := ev.Emit(testevent.Data{EventName: FailedEvent, Target: target, Payload: nil}); err != nil { + return fmt.Errorf("failed to emit finished event: %v", err) + } + return fmt.Errorf("target failed") + } else { + if err := ev.Emit(testevent.Data{EventName: FinishedEvent, Target: target, Payload: nil}); err != nil { + return fmt.Errorf("failed to emit failed event: %v", err) + } + } + return nil + } + if err := ev.Emit(testevent.Data{EventName: StepRunningEvent}); err != nil { + return fmt.Errorf("failed to emit failed event: %v", err) + } + res := teststeps.ForEachTarget(Name, ctx, ch, f) + if err := ev.Emit(testevent.Data{EventName: StepFinishedEvent}); err != nil { + return fmt.Errorf("failed to emit failed event: %v", err) + } + return res +} + +// ValidateParameters validates the parameters associated to the TestStep +func (ts *Step) ValidateParameters(params test.TestStepParameters) error { + targetsToFail := params.GetOne(FailTargetsParam).String() + if len(targetsToFail) > 0 { + for _, t := range strings.Split(targetsToFail, ",") { + ts.failTargets[t] = true + } + } + targetsToDelay := params.GetOne(DelayTargetsParam).String() + if len(targetsToDelay) > 0 { + for _, e := range strings.Split(targetsToDelay, ",") { + kv := strings.Split(e, "=") + if len(kv) != 2 { + continue + } + v, err := strconv.Atoi(kv[1]) + if err != nil { + return fmt.Errorf("invalid FailTargets: %w", err) + } + ts.delayTargets[kv[0]] = time.Duration(v) * time.Millisecond + } + } + if params.GetOne(FailPctParam).String() != "" { + if pct, err := params.GetInt(FailPctParam); err == nil { + ts.failPct = pct + } else { + return fmt.Errorf("invalid FailPct: %w", err) + } + } + return nil +} + +// Resume tries to resume a previously interrupted test step. TestTestStep +// cannot resume. +func (ts *Step) Resume(ctx statectx.Context, ch test.TestStepChannels, _ test.TestStepParameters, ev testevent.EmitterFetcher) error { + return &cerrors.ErrResumeNotSupported{StepName: Name} +} + +// CanResume tells whether this step is able to resume. +func (ts *Step) CanResume() bool { + return false +} + +// New initializes and returns a new TestStep. +func New() test.TestStep { + return &Step{ + failTargets: make(map[string]bool), + delayTargets: make(map[string]time.Duration), + } +} + +// Load returns the name, factory and events which are needed to register the step. +func Load() (string, test.TestStepFactory, []event.Name) { + return Name, New, Events +}