Skip to content

Commit

Permalink
bugfix: fix deadlock on prober start
Browse files Browse the repository at this point in the history
t1: event prober start and failed load bpf, leave the refcnt == 1
t2: metric prober start; refcnt == 2
    start perf reader failed and call stop
    deadlock on stop reentrant lock

Signed-off-by: bingshen.wbs <[email protected]>
  • Loading branch information
BSWANG committed Apr 17, 2024
1 parent 5b94e7f commit 240e019
Showing 1 changed file with 9 additions and 4 deletions.
13 changes: 9 additions & 4 deletions pkg/exporter/probe/tracekernel/tracekernel.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,7 @@ type kernelLatencyProbe struct {
metricsLock sync.RWMutex
}

func (p *kernelLatencyProbe) stop(_ context.Context, probeType probe.Type) error {
p.lock.Lock()
defer p.lock.Unlock()
func (p *kernelLatencyProbe) stopLocked(probeType probe.Type) error {
if p.refcnt[probeType] == 0 {
return fmt.Errorf("probe %s never start", probeType)
}
Expand All @@ -141,6 +139,12 @@ func (p *kernelLatencyProbe) stop(_ context.Context, probeType probe.Type) error
return nil
}

func (p *kernelLatencyProbe) stop(_ context.Context, probeType probe.Type) error {
p.lock.Lock()
defer p.lock.Unlock()
return p.stopLocked(probeType)
}

func (p *kernelLatencyProbe) cleanup() error {
if p.perfReader != nil {
p.perfReader.Close()
Expand Down Expand Up @@ -182,6 +186,7 @@ func (p *kernelLatencyProbe) start(ctx context.Context, probeType probe.Type) (e
p.refcnt[probeType]++
if p.totalReferenceCountLocked() == 1 {
if err = p.loadAndAttachBPF(); err != nil {
p.refcnt[probeType]--
log.Errorf("%s failed load and attach bpf, err: %v", probeName, err)
_ = p.cleanup()
return fmt.Errorf("%s failed load bpf: %w", probeName, err)
Expand All @@ -192,7 +197,7 @@ func (p *kernelLatencyProbe) start(ctx context.Context, probeType probe.Type) (e
p.perfReader, err = perf.NewReader(p.objs.bpfMaps.InspKlatencyEvent, int(unsafe.Sizeof(bpfInspKlEventT{})))
if err != nil {
log.Errorf("%s failed create perf reader, err: %v", probeName, err)
_ = p.stop(ctx, probeType)
_ = p.stopLocked(probeType)
return fmt.Errorf("%s failed create bpf reader: %w", probeName, err)
}

Expand Down

0 comments on commit 240e019

Please sign in to comment.