Skip to content
This repository has been archived by the owner on Jun 27, 2024. It is now read-only.

fix: improve destroy signal handling #72

Merged
merged 1 commit into from
May 20, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 33 additions & 6 deletions pool/static_pool/static_pool.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,12 @@ func (sp *Pool) QueueSize() uint64 {
// Destroy all underlying stack (but let them complete the task).
func (sp *Pool) Destroy(ctx context.Context) {
sp.log.Info("destroy signal received", zap.Duration("timeout", sp.cfg.DestroyTimeout))
ctx, cancel := context.WithTimeout(ctx, sp.cfg.DestroyTimeout)
defer cancel()
var cancel context.CancelFunc
_, ok := ctx.Deadline()
if !ok {
ctx, cancel = context.WithTimeout(ctx, sp.cfg.DestroyTimeout)
defer cancel()
}
sp.ww.Destroy(ctx)
atomic.StoreUint64(&sp.queue, 0)
}
Expand Down Expand Up @@ -233,7 +237,13 @@ func (sp *Pool) stopWorker(w *worker.Process) {
w.State().Transition(fsm.StateInvalid)
err := w.Stop()
if err != nil {
sp.log.Warn("user requested worker to be stopped", zap.String("reason", "user event"), zap.Int64("pid", w.Pid()), zap.String("internal_event_name", events.EventWorkerError.String()), zap.Error(err))
sp.log.Warn(
"user requested worker to be stopped",
zap.String("reason", "user event"),
zap.Int64("pid", w.Pid()),
zap.String("internal_event_name", events.EventWorkerError.String()),
zap.Error(err),
)
}
}

Expand All @@ -243,7 +253,12 @@ func (sp *Pool) takeWorker(ctxGetFree context.Context, op errors.Op) (*worker.Pr
if err != nil {
// if the error is of kind NoFreeWorkers, it means, that we can't get worker from the stack during the allocate timeout
if errors.Is(errors.NoFreeWorkers, err) {
sp.log.Error("no free workers in the pool, wait timeout exceed", zap.String("reason", "no free workers"), zap.String("internal_event_name", events.EventNoFreeWorkers.String()), zap.Error(err))
sp.log.Error(
"no free workers in the pool, wait timeout exceed",
zap.String("reason", "no free workers"),
zap.String("internal_event_name", events.EventNoFreeWorkers.String()),
zap.Error(err),
)
return nil, errors.E(op, err)
}
// else if err not nil - return error
Expand Down Expand Up @@ -273,7 +288,13 @@ func (sp *Pool) execDebug(p *payload.Payload) (*payload.Payload, error) {
// destroy the worker
err = sw.Stop()
if err != nil {
sp.log.Debug("debug mode: worker stopped", zap.String("reason", "worker error"), zap.Int64("pid", sw.Pid()), zap.String("internal_event_name", events.EventWorkerError.String()), zap.Error(err))
sp.log.Debug(
"debug mode: worker stopped",
zap.String("reason", "worker error"),
zap.Int64("pid", sw.Pid()),
zap.String("internal_event_name", events.EventWorkerError.String()),
zap.Error(err),
)
return nil, err
}

Expand All @@ -300,7 +321,13 @@ func (sp *Pool) execDebugWithTTL(ctx context.Context, p *payload.Payload) (*payl

err = sw.Stop()
if err != nil {
sp.log.Debug("debug mode: worker stopped", zap.String("reason", "worker error"), zap.Int64("pid", sw.Pid()), zap.String("internal_event_name", events.EventWorkerError.String()), zap.Error(err))
sp.log.Debug(
"debug mode: worker stopped",
zap.String("reason", "worker error"),
zap.Int64("pid", sw.Pid()),
zap.String("internal_event_name", events.EventWorkerError.String()),
zap.Error(err),
)
return nil, err
}

Expand Down
2 changes: 1 addition & 1 deletion worker_watcher/worker_watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ func (ww *WorkerWatcher) Destroy(ctx context.Context) {
return
case <-ctx.Done():
// drain channel
_, _ = ww.container.Pop(ctx)
ww.container.Drain()
// kill workers
ww.Lock()
wg := &sync.WaitGroup{}
Expand Down